2024-12-09 12:15:38,674 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-12-09 12:15:38,708 main DEBUG Took 0.021215 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-09 12:15:38,709 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-09 12:15:38,709 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-09 12:15:38,710 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-09 12:15:38,712 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 12:15:38,723 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-09 12:15:38,755 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 12:15:38,757 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 12:15:38,758 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 12:15:38,759 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 12:15:38,762 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 12:15:38,763 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 12:15:38,764 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 12:15:38,764 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 12:15:38,764 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 12:15:38,765 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 12:15:38,766 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 12:15:38,769 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 12:15:38,771 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 12:15:38,771 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 12:15:38,772 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 12:15:38,773 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 12:15:38,774 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 12:15:38,774 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 12:15:38,776 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 12:15:38,777 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 12:15:38,778 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 12:15:38,779 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 12:15:38,781 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 12:15:38,783 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 12:15:38,784 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 12:15:38,784 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-09 12:15:38,786 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 12:15:38,788 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-09 12:15:38,790 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-09 12:15:38,791 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-09 12:15:38,792 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-09 12:15:38,792 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-09 12:15:38,817 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-09 12:15:38,821 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-09 12:15:38,824 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-09 12:15:38,825 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-09 12:15:38,825 main DEBUG createAppenders(={Console}) 2024-12-09 12:15:38,826 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-12-09 12:15:38,826 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-12-09 12:15:38,827 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-12-09 12:15:38,828 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-09 12:15:38,828 main DEBUG OutputStream closed 2024-12-09 12:15:38,828 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-09 12:15:38,829 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-09 12:15:38,829 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-12-09 12:15:38,952 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-09 12:15:38,955 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-09 12:15:38,956 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-09 12:15:38,958 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-09 12:15:38,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-09 12:15:38,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-09 12:15:38,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-09 12:15:38,960 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-09 12:15:38,960 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-09 12:15:38,961 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-09 12:15:38,961 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-09 12:15:38,962 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-09 12:15:38,962 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-09 12:15:38,963 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-09 12:15:38,963 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-09 12:15:38,964 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-09 12:15:38,964 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-09 12:15:38,965 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-09 12:15:38,968 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09 12:15:38,969 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-12-09 12:15:38,969 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-09 12:15:38,970 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-12-09T12:15:38,991 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-12-09 12:15:38,996 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-09 12:15:38,996 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09T12:15:39,468 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023 2024-12-09T12:15:39,509 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-09T12:15:39,536 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/cluster_807fc2e9-136d-639e-43a2-3199c5f61929, deleteOnExit=true 2024-12-09T12:15:39,536 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-09T12:15:39,538 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/test.cache.data in system properties and HBase conf 2024-12-09T12:15:39,538 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/hadoop.tmp.dir in system properties and HBase conf 2024-12-09T12:15:39,540 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/hadoop.log.dir in system properties and HBase conf 2024-12-09T12:15:39,541 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-09T12:15:39,541 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-09T12:15:39,542 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-09T12:15:39,689 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-09T12:15:39,867 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T12:15:39,873 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-09T12:15:39,874 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-09T12:15:39,874 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-09T12:15:39,875 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T12:15:39,876 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-09T12:15:39,876 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-09T12:15:39,877 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T12:15:39,877 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T12:15:39,878 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-09T12:15:39,878 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/nfs.dump.dir in system properties and HBase conf 2024-12-09T12:15:39,879 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/java.io.tmpdir in system properties and HBase conf 2024-12-09T12:15:39,879 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T12:15:39,880 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-09T12:15:39,880 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-09T12:15:41,472 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-09T12:15:41,750 INFO [Time-limited test {}] log.Log(170): Logging initialized @4199ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-09T12:15:41,884 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T12:15:42,004 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T12:15:42,132 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T12:15:42,133 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T12:15:42,134 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-09T12:15:42,169 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T12:15:42,185 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@952689a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/hadoop.log.dir/,AVAILABLE} 2024-12-09T12:15:42,188 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@378c34a2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T12:15:42,649 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@302cfe51{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/java.io.tmpdir/jetty-localhost-41915-hadoop-hdfs-3_4_1-tests_jar-_-any-7626430094529043757/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T12:15:42,671 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@c1eb5a9{HTTP/1.1, (http/1.1)}{localhost:41915} 2024-12-09T12:15:42,672 INFO [Time-limited test {}] server.Server(415): Started @5122ms 2024-12-09T12:15:43,542 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T12:15:43,557 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T12:15:43,563 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T12:15:43,563 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T12:15:43,564 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-09T12:15:43,565 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@18a3047c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/hadoop.log.dir/,AVAILABLE} 2024-12-09T12:15:43,566 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@62c0eab0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T12:15:43,734 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1a0ce16d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/java.io.tmpdir/jetty-localhost-39851-hadoop-hdfs-3_4_1-tests_jar-_-any-11144426020458350868/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T12:15:43,735 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1cafe9ff{HTTP/1.1, (http/1.1)}{localhost:39851} 2024-12-09T12:15:43,735 INFO [Time-limited test {}] server.Server(415): Started @6185ms 2024-12-09T12:15:43,805 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T12:15:44,139 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T12:15:44,153 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T12:15:44,212 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T12:15:44,212 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T12:15:44,212 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T12:15:44,224 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@414e5e95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/hadoop.log.dir/,AVAILABLE} 2024-12-09T12:15:44,225 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a49f6d7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T12:15:44,495 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4caee23f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/java.io.tmpdir/jetty-localhost-39225-hadoop-hdfs-3_4_1-tests_jar-_-any-15119991270885807679/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T12:15:44,496 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@178c0f89{HTTP/1.1, (http/1.1)}{localhost:39225} 2024-12-09T12:15:44,497 INFO [Time-limited test {}] server.Server(415): Started @6947ms 2024-12-09T12:15:44,500 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T12:15:44,699 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T12:15:44,713 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T12:15:44,730 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T12:15:44,730 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T12:15:44,731 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-09T12:15:44,733 WARN [Thread-102 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/cluster_807fc2e9-136d-639e-43a2-3199c5f61929/data/data1/current/BP-496458314-172.17.0.3-1733746541015/current, will proceed with Du for space computation calculation, 2024-12-09T12:15:44,734 WARN [Thread-104 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/cluster_807fc2e9-136d-639e-43a2-3199c5f61929/data/data2/current/BP-496458314-172.17.0.3-1733746541015/current, will proceed with Du for space computation calculation, 2024-12-09T12:15:44,738 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@60467e4f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/hadoop.log.dir/,AVAILABLE} 2024-12-09T12:15:44,739 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7e0151e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T12:15:44,818 WARN [Thread-109 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/cluster_807fc2e9-136d-639e-43a2-3199c5f61929/data/data3/current/BP-496458314-172.17.0.3-1733746541015/current, will proceed with Du for space computation calculation, 2024-12-09T12:15:44,820 WARN [Thread-110 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/cluster_807fc2e9-136d-639e-43a2-3199c5f61929/data/data4/current/BP-496458314-172.17.0.3-1733746541015/current, will proceed with Du for space computation calculation, 2024-12-09T12:15:44,844 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T12:15:44,968 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3126f578a87183c5 with lease ID 0xffbfd0104b885035: Processing first storage report for DS-e1bc5ec1-d599-496b-8be4-5493e5661bf9 from datanode DatanodeRegistration(127.0.0.1:33961, datanodeUuid=d514a274-aa3e-4ffc-b732-42d3b15a826e, infoPort=41475, infoSecurePort=0, ipcPort=34885, storageInfo=lv=-57;cid=testClusterID;nsid=89631050;c=1733746541015) 2024-12-09T12:15:44,969 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3126f578a87183c5 with lease ID 0xffbfd0104b885035: from storage DS-e1bc5ec1-d599-496b-8be4-5493e5661bf9 node DatanodeRegistration(127.0.0.1:33961, datanodeUuid=d514a274-aa3e-4ffc-b732-42d3b15a826e, infoPort=41475, infoSecurePort=0, ipcPort=34885, storageInfo=lv=-57;cid=testClusterID;nsid=89631050;c=1733746541015), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-09T12:15:44,973 WARN [Thread-83 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T12:15:44,987 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@459058f9{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/java.io.tmpdir/jetty-localhost-34057-hadoop-hdfs-3_4_1-tests_jar-_-any-11102301357076662741/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T12:15:44,972 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3126f578a87183c5 with lease ID 0xffbfd0104b885035: Processing first storage report for DS-65330786-8904-41c6-b5f1-ce89077f5705 from datanode DatanodeRegistration(127.0.0.1:33961, datanodeUuid=d514a274-aa3e-4ffc-b732-42d3b15a826e, infoPort=41475, infoSecurePort=0, ipcPort=34885, storageInfo=lv=-57;cid=testClusterID;nsid=89631050;c=1733746541015) 2024-12-09T12:15:44,989 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3126f578a87183c5 with lease ID 0xffbfd0104b885035: from storage DS-65330786-8904-41c6-b5f1-ce89077f5705 node DatanodeRegistration(127.0.0.1:33961, datanodeUuid=d514a274-aa3e-4ffc-b732-42d3b15a826e, infoPort=41475, infoSecurePort=0, ipcPort=34885, storageInfo=lv=-57;cid=testClusterID;nsid=89631050;c=1733746541015), blocks: 0, hasStaleStorage: false, processing time: 17 msecs, invalidatedBlocks: 0 2024-12-09T12:15:44,999 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@38c64c15{HTTP/1.1, (http/1.1)}{localhost:34057} 2024-12-09T12:15:44,999 INFO [Time-limited test {}] server.Server(415): Started @7449ms 2024-12-09T12:15:45,000 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5e0015de4ae996ed with lease ID 0xffbfd0104b885036: Processing first storage report for DS-0456a585-8dc2-4cb5-ae72-660bf02f0e80 from datanode DatanodeRegistration(127.0.0.1:39373, datanodeUuid=dd2670ca-2643-4a2a-ad7d-2be859ee2d51, infoPort=42689, infoSecurePort=0, ipcPort=39559, storageInfo=lv=-57;cid=testClusterID;nsid=89631050;c=1733746541015) 2024-12-09T12:15:45,000 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5e0015de4ae996ed with lease ID 0xffbfd0104b885036: from storage DS-0456a585-8dc2-4cb5-ae72-660bf02f0e80 node DatanodeRegistration(127.0.0.1:39373, datanodeUuid=dd2670ca-2643-4a2a-ad7d-2be859ee2d51, infoPort=42689, infoSecurePort=0, ipcPort=39559, storageInfo=lv=-57;cid=testClusterID;nsid=89631050;c=1733746541015), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T12:15:45,000 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5e0015de4ae996ed with lease ID 0xffbfd0104b885036: Processing first storage report for DS-897a5114-6185-4ec9-9ba7-18a88b057ee7 from datanode DatanodeRegistration(127.0.0.1:39373, datanodeUuid=dd2670ca-2643-4a2a-ad7d-2be859ee2d51, infoPort=42689, infoSecurePort=0, ipcPort=39559, storageInfo=lv=-57;cid=testClusterID;nsid=89631050;c=1733746541015) 2024-12-09T12:15:45,001 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5e0015de4ae996ed with lease ID 0xffbfd0104b885036: from storage DS-897a5114-6185-4ec9-9ba7-18a88b057ee7 node DatanodeRegistration(127.0.0.1:39373, datanodeUuid=dd2670ca-2643-4a2a-ad7d-2be859ee2d51, infoPort=42689, infoSecurePort=0, ipcPort=39559, storageInfo=lv=-57;cid=testClusterID;nsid=89631050;c=1733746541015), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T12:15:45,002 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T12:15:45,273 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/cluster_807fc2e9-136d-639e-43a2-3199c5f61929/data/data5/current/BP-496458314-172.17.0.3-1733746541015/current, will proceed with Du for space computation calculation, 2024-12-09T12:15:45,295 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/cluster_807fc2e9-136d-639e-43a2-3199c5f61929/data/data6/current/BP-496458314-172.17.0.3-1733746541015/current, will proceed with Du for space computation calculation, 2024-12-09T12:15:45,407 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T12:15:45,420 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x84e7f40bd7f48c3f with lease ID 0xffbfd0104b885037: Processing first storage report for DS-be4b45f7-c1d4-4431-b834-2678e63cb272 from datanode DatanodeRegistration(127.0.0.1:33963, datanodeUuid=1d97f8cf-de3d-46ee-aa97-243653d40d28, infoPort=41543, infoSecurePort=0, ipcPort=46779, storageInfo=lv=-57;cid=testClusterID;nsid=89631050;c=1733746541015) 2024-12-09T12:15:45,420 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x84e7f40bd7f48c3f with lease ID 0xffbfd0104b885037: from storage DS-be4b45f7-c1d4-4431-b834-2678e63cb272 node DatanodeRegistration(127.0.0.1:33963, datanodeUuid=1d97f8cf-de3d-46ee-aa97-243653d40d28, infoPort=41543, infoSecurePort=0, ipcPort=46779, storageInfo=lv=-57;cid=testClusterID;nsid=89631050;c=1733746541015), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T12:15:45,421 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x84e7f40bd7f48c3f with lease ID 0xffbfd0104b885037: Processing first storage report for DS-8eee0933-af2d-453c-82a9-6a16448ecff6 from datanode DatanodeRegistration(127.0.0.1:33963, datanodeUuid=1d97f8cf-de3d-46ee-aa97-243653d40d28, infoPort=41543, infoSecurePort=0, ipcPort=46779, storageInfo=lv=-57;cid=testClusterID;nsid=89631050;c=1733746541015) 2024-12-09T12:15:45,421 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x84e7f40bd7f48c3f with lease ID 0xffbfd0104b885037: from storage DS-8eee0933-af2d-453c-82a9-6a16448ecff6 node DatanodeRegistration(127.0.0.1:33963, datanodeUuid=1d97f8cf-de3d-46ee-aa97-243653d40d28, infoPort=41543, infoSecurePort=0, ipcPort=46779, storageInfo=lv=-57;cid=testClusterID;nsid=89631050;c=1733746541015), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T12:15:45,685 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023 2024-12-09T12:15:45,836 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/cluster_807fc2e9-136d-639e-43a2-3199c5f61929/zookeeper_0, clientPort=60469, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/cluster_807fc2e9-136d-639e-43a2-3199c5f61929/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2ae888f8-4a69-815a-14d8-15c5b1a87023/cluster_807fc2e9-136d-639e-43a2-3199c5f61929/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-09T12:15:45,850 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60469 2024-12-09T12:15:45,869 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T12:15:45,873 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T12:15:46,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741825_1001 (size=7) 2024-12-09T12:15:46,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741825_1001 (size=7) 2024-12-09T12:15:46,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741825_1001 (size=7) 2024-12-09T12:15:46,733 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee with version=8 2024-12-09T12:15:46,733 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/hbase-staging 2024-12-09T12:15:46,856 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-09T12:15:47,201 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f15dd41c88ad:0 server-side Connection retries=45 2024-12-09T12:15:47,215 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T12:15:47,217 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T12:15:47,248 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T12:15:47,248 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T12:15:47,249 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T12:15:47,545 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-09T12:15:47,642 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-09T12:15:47,662 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-09T12:15:47,676 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T12:15:47,723 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 8238 (auto-detected) 2024-12-09T12:15:47,724 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-09T12:15:47,770 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42519 2024-12-09T12:15:47,837 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42519 connecting to ZooKeeper ensemble=127.0.0.1:60469 2024-12-09T12:15:47,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:425190x0, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T12:15:47,977 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42519-0x1012b1c64ac0000 connected 2024-12-09T12:15:48,153 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T12:15:48,168 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T12:15:48,199 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T12:15:48,206 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee, hbase.cluster.distributed=false 2024-12-09T12:15:48,269 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T12:15:48,298 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42519 2024-12-09T12:15:48,304 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42519 2024-12-09T12:15:48,305 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42519 2024-12-09T12:15:48,306 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42519 2024-12-09T12:15:48,313 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42519 2024-12-09T12:15:48,533 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f15dd41c88ad:0 server-side Connection retries=45 2024-12-09T12:15:48,535 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T12:15:48,535 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T12:15:48,536 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T12:15:48,536 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T12:15:48,536 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T12:15:48,539 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T12:15:48,541 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T12:15:48,542 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38111 2024-12-09T12:15:48,544 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38111 connecting to ZooKeeper ensemble=127.0.0.1:60469 2024-12-09T12:15:48,545 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T12:15:48,549 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T12:15:48,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:381110x0, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T12:15:48,567 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38111-0x1012b1c64ac0001 connected 2024-12-09T12:15:48,571 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T12:15:48,575 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T12:15:48,584 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T12:15:48,586 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T12:15:48,593 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T12:15:48,601 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38111 2024-12-09T12:15:48,604 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38111 2024-12-09T12:15:48,605 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38111 2024-12-09T12:15:48,606 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38111 2024-12-09T12:15:48,610 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38111 2024-12-09T12:15:48,635 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f15dd41c88ad:0 server-side Connection retries=45 2024-12-09T12:15:48,636 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T12:15:48,636 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T12:15:48,637 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T12:15:48,637 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T12:15:48,637 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T12:15:48,637 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T12:15:48,638 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T12:15:48,639 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:33783 2024-12-09T12:15:48,643 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33783 connecting to ZooKeeper ensemble=127.0.0.1:60469 2024-12-09T12:15:48,646 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T12:15:48,649 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T12:15:48,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:337830x0, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T12:15:48,658 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:337830x0, quorum=127.0.0.1:60469, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T12:15:48,659 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T12:15:48,667 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33783-0x1012b1c64ac0002 connected 2024-12-09T12:15:48,677 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T12:15:48,678 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33783-0x1012b1c64ac0002, quorum=127.0.0.1:60469, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T12:15:48,681 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33783-0x1012b1c64ac0002, quorum=127.0.0.1:60469, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T12:15:48,691 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33783 2024-12-09T12:15:48,692 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33783 2024-12-09T12:15:48,693 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33783 2024-12-09T12:15:48,699 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33783 2024-12-09T12:15:48,702 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33783 2024-12-09T12:15:48,728 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f15dd41c88ad:0 server-side Connection retries=45 2024-12-09T12:15:48,728 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T12:15:48,729 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T12:15:48,729 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T12:15:48,729 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T12:15:48,729 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T12:15:48,729 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T12:15:48,730 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T12:15:48,731 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36579 2024-12-09T12:15:48,733 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36579 connecting to ZooKeeper ensemble=127.0.0.1:60469 2024-12-09T12:15:48,734 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T12:15:48,737 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T12:15:48,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:365790x0, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T12:15:48,745 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:365790x0, quorum=127.0.0.1:60469, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T12:15:48,746 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T12:15:48,751 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36579-0x1012b1c64ac0003 connected 2024-12-09T12:15:48,762 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T12:15:48,765 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T12:15:48,769 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T12:15:48,772 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36579 2024-12-09T12:15:48,772 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36579 2024-12-09T12:15:48,774 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36579 2024-12-09T12:15:48,786 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36579 2024-12-09T12:15:48,787 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36579 2024-12-09T12:15:48,812 DEBUG [M:0;f15dd41c88ad:42519 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f15dd41c88ad:42519 2024-12-09T12:15:48,813 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f15dd41c88ad,42519,1733746546940 2024-12-09T12:15:48,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T12:15:48,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T12:15:48,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T12:15:48,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33783-0x1012b1c64ac0002, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T12:15:48,827 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f15dd41c88ad,42519,1733746546940 2024-12-09T12:15:48,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T12:15:48,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T12:15:48,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T12:15:48,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T12:15:48,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33783-0x1012b1c64ac0002, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T12:15:48,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33783-0x1012b1c64ac0002, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T12:15:48,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T12:15:48,901 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T12:15:48,903 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f15dd41c88ad,42519,1733746546940 from backup master directory 2024-12-09T12:15:48,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f15dd41c88ad,42519,1733746546940 2024-12-09T12:15:48,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T12:15:48,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T12:15:48,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T12:15:48,910 WARN [master/f15dd41c88ad:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T12:15:48,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33783-0x1012b1c64ac0002, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T12:15:48,910 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f15dd41c88ad,42519,1733746546940 2024-12-09T12:15:48,934 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-09T12:15:48,953 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-09T12:15:49,048 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/hbase.id] with ID: bd14337c-7e24-4686-a8c1-6c16b6d09ea0 2024-12-09T12:15:49,048 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/.tmp/hbase.id 2024-12-09T12:15:49,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741826_1002 (size=42) 2024-12-09T12:15:49,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741826_1002 (size=42) 2024-12-09T12:15:49,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741826_1002 (size=42) 2024-12-09T12:15:49,081 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/.tmp/hbase.id]:[hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/hbase.id] 2024-12-09T12:15:49,168 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T12:15:49,175 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-09T12:15:49,204 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 25ms. 2024-12-09T12:15:49,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T12:15:49,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T12:15:49,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33783-0x1012b1c64ac0002, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T12:15:49,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T12:15:49,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741827_1003 (size=196) 2024-12-09T12:15:49,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741827_1003 (size=196) 2024-12-09T12:15:49,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741827_1003 (size=196) 2024-12-09T12:15:49,295 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T12:15:49,298 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-09T12:15:49,321 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T12:15:49,329 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T12:15:49,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741828_1004 (size=1189) 2024-12-09T12:15:49,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741828_1004 (size=1189) 2024-12-09T12:15:49,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741828_1004 (size=1189) 2024-12-09T12:15:49,475 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/MasterData/data/master/store 2024-12-09T12:15:49,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741829_1005 (size=34) 2024-12-09T12:15:49,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741829_1005 (size=34) 2024-12-09T12:15:49,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741829_1005 (size=34) 2024-12-09T12:15:49,543 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-09T12:15:49,548 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:49,549 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T12:15:49,550 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T12:15:49,550 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T12:15:49,553 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T12:15:49,553 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T12:15:49,554 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T12:15:49,555 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733746549549Disabling compacts and flushes for region at 1733746549549Disabling writes for close at 1733746549553 (+4 ms)Writing region close event to WAL at 1733746549554 (+1 ms)Closed at 1733746549554 2024-12-09T12:15:49,558 WARN [master/f15dd41c88ad:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/MasterData/data/master/store/.initializing 2024-12-09T12:15:49,558 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/MasterData/WALs/f15dd41c88ad,42519,1733746546940 2024-12-09T12:15:49,579 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T12:15:49,599 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f15dd41c88ad%2C42519%2C1733746546940, suffix=, logDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/MasterData/WALs/f15dd41c88ad,42519,1733746546940, archiveDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/MasterData/oldWALs, maxLogs=10 2024-12-09T12:15:49,632 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/MasterData/WALs/f15dd41c88ad,42519,1733746546940/f15dd41c88ad%2C42519%2C1733746546940.1733746549605, exclude list is [], retry=0 2024-12-09T12:15:49,660 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39373,DS-0456a585-8dc2-4cb5-ae72-660bf02f0e80,DISK] 2024-12-09T12:15:49,660 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33961,DS-e1bc5ec1-d599-496b-8be4-5493e5661bf9,DISK] 2024-12-09T12:15:49,660 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33963,DS-be4b45f7-c1d4-4431-b834-2678e63cb272,DISK] 2024-12-09T12:15:49,664 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-09T12:15:49,722 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/MasterData/WALs/f15dd41c88ad,42519,1733746546940/f15dd41c88ad%2C42519%2C1733746546940.1733746549605 2024-12-09T12:15:49,724 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41475:41475),(127.0.0.1/127.0.0.1:41543:41543),(127.0.0.1/127.0.0.1:42689:42689)] 2024-12-09T12:15:49,724 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-09T12:15:49,725 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:49,730 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T12:15:49,732 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T12:15:49,795 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T12:15:49,831 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-09T12:15:49,836 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:49,843 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T12:15:49,846 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T12:15:49,851 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-09T12:15:49,851 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:49,853 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:49,854 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T12:15:49,857 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-09T12:15:49,857 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:49,859 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:49,860 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T12:15:49,863 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-09T12:15:49,864 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:49,867 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:49,868 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T12:15:49,875 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T12:15:49,878 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T12:15:49,914 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T12:15:49,915 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T12:15:49,924 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T12:15:49,943 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T12:15:49,976 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T12:15:49,979 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61479431, jitterRate=-0.0838850885629654}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T12:15:49,989 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733746549753Initializing all the Stores at 1733746549756 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733746549757 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746549762 (+5 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746549763 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746549763Cleaning up temporary data from old regions at 1733746549915 (+152 ms)Region opened successfully at 1733746549989 (+74 ms) 2024-12-09T12:15:49,994 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-09T12:15:50,050 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f848959, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f15dd41c88ad/172.17.0.3:0 2024-12-09T12:15:50,092 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-09T12:15:50,106 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-09T12:15:50,106 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-09T12:15:50,110 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T12:15:50,112 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-09T12:15:50,118 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-09T12:15:50,118 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-09T12:15:50,169 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-09T12:15:50,183 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-09T12:15:50,186 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-09T12:15:50,190 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-09T12:15:50,191 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-09T12:15:50,194 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-09T12:15:50,197 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-09T12:15:50,203 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-09T12:15:50,206 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-09T12:15:50,209 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-09T12:15:50,218 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-09T12:15:50,261 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-09T12:15:50,263 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-09T12:15:50,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T12:15:50,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T12:15:50,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T12:15:50,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33783-0x1012b1c64ac0002, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T12:15:50,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33783-0x1012b1c64ac0002, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T12:15:50,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T12:15:50,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T12:15:50,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T12:15:50,281 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f15dd41c88ad,42519,1733746546940, sessionid=0x1012b1c64ac0000, setting cluster-up flag (Was=false) 2024-12-09T12:15:50,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T12:15:50,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T12:15:50,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T12:15:50,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33783-0x1012b1c64ac0002, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T12:15:50,324 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-09T12:15:50,328 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f15dd41c88ad,42519,1733746546940 2024-12-09T12:15:50,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T12:15:50,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33783-0x1012b1c64ac0002, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T12:15:50,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T12:15:50,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T12:15:50,366 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-09T12:15:50,375 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f15dd41c88ad,42519,1733746546940 2024-12-09T12:15:50,387 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-09T12:15:50,405 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(746): ClusterId : bd14337c-7e24-4686-a8c1-6c16b6d09ea0 2024-12-09T12:15:50,409 DEBUG [RS:2;f15dd41c88ad:36579 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T12:15:50,410 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer(746): ClusterId : bd14337c-7e24-4686-a8c1-6c16b6d09ea0 2024-12-09T12:15:50,410 DEBUG [RS:0;f15dd41c88ad:38111 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T12:15:50,411 INFO [RS:1;f15dd41c88ad:33783 {}] regionserver.HRegionServer(746): ClusterId : bd14337c-7e24-4686-a8c1-6c16b6d09ea0 2024-12-09T12:15:50,411 DEBUG [RS:1;f15dd41c88ad:33783 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T12:15:50,422 DEBUG [RS:1;f15dd41c88ad:33783 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T12:15:50,423 DEBUG [RS:1;f15dd41c88ad:33783 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T12:15:50,424 DEBUG [RS:2;f15dd41c88ad:36579 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T12:15:50,424 DEBUG [RS:2;f15dd41c88ad:36579 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T12:15:50,424 DEBUG [RS:0;f15dd41c88ad:38111 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T12:15:50,425 DEBUG [RS:0;f15dd41c88ad:38111 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T12:15:50,432 DEBUG [RS:0;f15dd41c88ad:38111 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T12:15:50,433 DEBUG [RS:0;f15dd41c88ad:38111 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@240f6122, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f15dd41c88ad/172.17.0.3:0 2024-12-09T12:15:50,434 DEBUG [RS:1;f15dd41c88ad:33783 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T12:15:50,436 DEBUG [RS:1;f15dd41c88ad:33783 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21bdaa3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f15dd41c88ad/172.17.0.3:0 2024-12-09T12:15:50,437 DEBUG [RS:2;f15dd41c88ad:36579 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T12:15:50,437 DEBUG [RS:2;f15dd41c88ad:36579 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2be19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f15dd41c88ad/172.17.0.3:0 2024-12-09T12:15:50,454 DEBUG [RS:2;f15dd41c88ad:36579 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;f15dd41c88ad:36579 2024-12-09T12:15:50,459 DEBUG [RS:0;f15dd41c88ad:38111 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f15dd41c88ad:38111 2024-12-09T12:15:50,460 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T12:15:50,461 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T12:15:50,461 DEBUG [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T12:15:50,461 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T12:15:50,461 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T12:15:50,462 DEBUG [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T12:15:50,464 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer(2659): reportForDuty to master=f15dd41c88ad,42519,1733746546940 with port=38111, startcode=1733746548467 2024-12-09T12:15:50,465 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(2659): reportForDuty to master=f15dd41c88ad,42519,1733746546940 with port=36579, startcode=1733746548727 2024-12-09T12:15:50,467 DEBUG [RS:1;f15dd41c88ad:33783 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;f15dd41c88ad:33783 2024-12-09T12:15:50,467 INFO [RS:1;f15dd41c88ad:33783 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T12:15:50,467 INFO [RS:1;f15dd41c88ad:33783 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T12:15:50,467 DEBUG [RS:1;f15dd41c88ad:33783 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T12:15:50,473 INFO [RS:1;f15dd41c88ad:33783 {}] regionserver.HRegionServer(2659): reportForDuty to master=f15dd41c88ad,42519,1733746546940 with port=33783, startcode=1733746548634 2024-12-09T12:15:50,483 DEBUG [RS:1;f15dd41c88ad:33783 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T12:15:50,486 DEBUG [RS:2;f15dd41c88ad:36579 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T12:15:50,488 DEBUG [RS:0;f15dd41c88ad:38111 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T12:15:50,529 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T12:15:50,540 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37215, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T12:15:50,541 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40773, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T12:15:50,543 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53611, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T12:15:50,548 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:15:50,551 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-09T12:15:50,555 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:15:50,556 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:15:50,563 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-09T12:15:50,569 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f15dd41c88ad,42519,1733746546940 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-09T12:15:50,586 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f15dd41c88ad:0, corePoolSize=5, maxPoolSize=5 2024-12-09T12:15:50,586 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f15dd41c88ad:0, corePoolSize=5, maxPoolSize=5 2024-12-09T12:15:50,586 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f15dd41c88ad:0, corePoolSize=5, maxPoolSize=5 2024-12-09T12:15:50,586 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f15dd41c88ad:0, corePoolSize=5, maxPoolSize=5 2024-12-09T12:15:50,586 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f15dd41c88ad:0, corePoolSize=10, maxPoolSize=10 2024-12-09T12:15:50,587 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,587 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f15dd41c88ad:0, corePoolSize=2, maxPoolSize=2 2024-12-09T12:15:50,587 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,592 DEBUG [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-09T12:15:50,592 DEBUG [RS:1;f15dd41c88ad:33783 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-09T12:15:50,592 WARN [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-09T12:15:50,592 WARN [RS:1;f15dd41c88ad:33783 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-09T12:15:50,593 DEBUG [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-09T12:15:50,593 WARN [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-09T12:15:50,611 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733746580611 2024-12-09T12:15:50,613 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-09T12:15:50,614 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-09T12:15:50,618 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T12:15:50,619 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-09T12:15:50,619 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-09T12:15:50,619 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-09T12:15:50,620 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-09T12:15:50,620 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-09T12:15:50,621 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:50,635 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:50,636 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-09T12:15:50,643 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-09T12:15:50,644 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-09T12:15:50,645 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-09T12:15:50,653 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-09T12:15:50,653 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-09T12:15:50,670 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f15dd41c88ad:0:becomeActiveMaster-HFileCleaner.large.0-1733746550660,5,FailOnTimeoutGroup] 2024-12-09T12:15:50,690 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f15dd41c88ad:0:becomeActiveMaster-HFileCleaner.small.0-1733746550670,5,FailOnTimeoutGroup] 2024-12-09T12:15:50,691 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:50,692 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-09T12:15:50,694 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(2659): reportForDuty to master=f15dd41c88ad,42519,1733746546940 with port=36579, startcode=1733746548727 2024-12-09T12:15:50,695 INFO [RS:1;f15dd41c88ad:33783 {}] regionserver.HRegionServer(2659): reportForDuty to master=f15dd41c88ad,42519,1733746546940 with port=33783, startcode=1733746548634 2024-12-09T12:15:50,696 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer(2659): reportForDuty to master=f15dd41c88ad,42519,1733746546940 with port=38111, startcode=1733746548467 2024-12-09T12:15:50,696 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:15:50,697 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:50,697 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:15:50,698 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:50,698 DEBUG [RS:1;f15dd41c88ad:33783 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-09T12:15:50,698 WARN [RS:1;f15dd41c88ad:33783 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 200 ms and then retrying. 2024-12-09T12:15:50,700 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f15dd41c88ad,38111,1733746548467 2024-12-09T12:15:50,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741831_1007 (size=1321) 2024-12-09T12:15:50,703 DEBUG [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-09T12:15:50,703 WARN [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 200 ms and then retrying. 2024-12-09T12:15:50,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741831_1007 (size=1321) 2024-12-09T12:15:50,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741831_1007 (size=1321) 2024-12-09T12:15:50,707 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] master.ServerManager(517): Registering regionserver=f15dd41c88ad,38111,1733746548467 2024-12-09T12:15:50,707 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-09T12:15:50,708 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:50,722 DEBUG [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:50,722 DEBUG [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36283 2024-12-09T12:15:50,723 DEBUG [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T12:15:50,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T12:15:50,735 DEBUG [RS:0;f15dd41c88ad:38111 {}] zookeeper.ZKUtil(111): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f15dd41c88ad,38111,1733746548467 2024-12-09T12:15:50,735 WARN [RS:0;f15dd41c88ad:38111 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T12:15:50,735 INFO [RS:0;f15dd41c88ad:38111 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T12:15:50,736 DEBUG [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,38111,1733746548467 2024-12-09T12:15:50,738 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f15dd41c88ad,38111,1733746548467] 2024-12-09T12:15:50,802 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T12:15:50,821 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T12:15:50,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741832_1008 (size=32) 2024-12-09T12:15:50,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741832_1008 (size=32) 2024-12-09T12:15:50,828 INFO [RS:0;f15dd41c88ad:38111 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T12:15:50,828 INFO [RS:0;f15dd41c88ad:38111 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:50,829 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T12:15:50,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741832_1008 (size=32) 2024-12-09T12:15:50,832 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:50,835 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T12:15:50,835 INFO [RS:0;f15dd41c88ad:38111 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T12:15:50,837 INFO [RS:0;f15dd41c88ad:38111 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:50,837 DEBUG [RS:0;f15dd41c88ad:38111 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,837 DEBUG [RS:0;f15dd41c88ad:38111 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,838 DEBUG [RS:0;f15dd41c88ad:38111 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,838 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T12:15:50,838 DEBUG [RS:0;f15dd41c88ad:38111 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,838 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:50,838 DEBUG [RS:0;f15dd41c88ad:38111 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,838 DEBUG [RS:0;f15dd41c88ad:38111 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0, corePoolSize=2, maxPoolSize=2 2024-12-09T12:15:50,838 DEBUG [RS:0;f15dd41c88ad:38111 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,838 DEBUG [RS:0;f15dd41c88ad:38111 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,839 DEBUG [RS:0;f15dd41c88ad:38111 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,839 DEBUG [RS:0;f15dd41c88ad:38111 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,839 DEBUG [RS:0;f15dd41c88ad:38111 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,839 DEBUG [RS:0;f15dd41c88ad:38111 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,839 DEBUG [RS:0;f15dd41c88ad:38111 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f15dd41c88ad:0, corePoolSize=3, maxPoolSize=3 2024-12-09T12:15:50,839 DEBUG [RS:0;f15dd41c88ad:38111 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f15dd41c88ad:0, corePoolSize=3, maxPoolSize=3 2024-12-09T12:15:50,840 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T12:15:50,840 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T12:15:50,843 INFO [RS:0;f15dd41c88ad:38111 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:50,843 INFO [RS:0;f15dd41c88ad:38111 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:50,843 INFO [RS:0;f15dd41c88ad:38111 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:50,843 INFO [RS:0;f15dd41c88ad:38111 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:50,843 INFO [RS:0;f15dd41c88ad:38111 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:50,843 INFO [RS:0;f15dd41c88ad:38111 {}] hbase.ChoreService(168): Chore ScheduledChore name=f15dd41c88ad,38111,1733746548467-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T12:15:50,844 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T12:15:50,844 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:50,845 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T12:15:50,845 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T12:15:50,850 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T12:15:50,850 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:50,851 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T12:15:50,851 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T12:15:50,854 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T12:15:50,854 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:50,855 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T12:15:50,856 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T12:15:50,857 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740 2024-12-09T12:15:50,858 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740 2024-12-09T12:15:50,862 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T12:15:50,862 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T12:15:50,863 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T12:15:50,866 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T12:15:50,874 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T12:15:50,876 INFO [RS:0;f15dd41c88ad:38111 {}] hbase.ChoreService(168): Chore ScheduledChore name=f15dd41c88ad,38111,1733746548467-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:50,877 INFO [RS:0;f15dd41c88ad:38111 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:50,877 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.Replication(171): f15dd41c88ad,38111,1733746548467 started 2024-12-09T12:15:50,882 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T12:15:50,887 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66285472, jitterRate=-0.01226949691772461}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T12:15:50,890 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733746550832Initializing all the Stores at 1733746550834 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733746550834Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733746550835 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746550835Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733746550835Cleaning up temporary data from old regions at 1733746550862 (+27 ms)Region opened successfully at 1733746550890 (+28 ms) 2024-12-09T12:15:50,890 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T12:15:50,890 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T12:15:50,890 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T12:15:50,890 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T12:15:50,890 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T12:15:50,896 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T12:15:50,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733746550890Disabling compacts and flushes for region at 1733746550890Disabling writes for close at 1733746550890Writing region close event to WAL at 1733746550895 (+5 ms)Closed at 1733746550896 (+1 ms) 2024-12-09T12:15:50,900 INFO [RS:1;f15dd41c88ad:33783 {}] regionserver.HRegionServer(2659): reportForDuty to master=f15dd41c88ad,42519,1733746546940 with port=33783, startcode=1733746548634 2024-12-09T12:15:50,902 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:50,902 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] master.ServerManager(517): Registering regionserver=f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:50,902 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T12:15:50,902 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-09T12:15:50,907 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(2659): reportForDuty to master=f15dd41c88ad,42519,1733746546940 with port=36579, startcode=1733746548727 2024-12-09T12:15:50,908 DEBUG [RS:1;f15dd41c88ad:33783 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:50,908 DEBUG [RS:1;f15dd41c88ad:33783 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36283 2024-12-09T12:15:50,908 DEBUG [RS:1;f15dd41c88ad:33783 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T12:15:50,909 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f15dd41c88ad,36579,1733746548727 2024-12-09T12:15:50,909 INFO [RS:0;f15dd41c88ad:38111 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:50,910 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] master.ServerManager(517): Registering regionserver=f15dd41c88ad,36579,1733746548727 2024-12-09T12:15:50,911 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer(1482): Serving as f15dd41c88ad,38111,1733746548467, RpcServer on f15dd41c88ad/172.17.0.3:38111, sessionid=0x1012b1c64ac0001 2024-12-09T12:15:50,912 DEBUG [RS:0;f15dd41c88ad:38111 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T12:15:50,913 DEBUG [RS:0;f15dd41c88ad:38111 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f15dd41c88ad,38111,1733746548467 2024-12-09T12:15:50,914 DEBUG [RS:0;f15dd41c88ad:38111 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f15dd41c88ad,38111,1733746548467' 2024-12-09T12:15:50,914 DEBUG [RS:0;f15dd41c88ad:38111 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T12:15:50,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T12:15:50,915 DEBUG [RS:1;f15dd41c88ad:33783 {}] zookeeper.ZKUtil(111): regionserver:33783-0x1012b1c64ac0002, quorum=127.0.0.1:60469, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:50,915 WARN [RS:1;f15dd41c88ad:33783 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T12:15:50,915 INFO [RS:1;f15dd41c88ad:33783 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T12:15:50,915 DEBUG [RS:1;f15dd41c88ad:33783 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:50,916 DEBUG [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:50,916 DEBUG [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36283 2024-12-09T12:15:50,916 DEBUG [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T12:15:50,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T12:15:50,918 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f15dd41c88ad,33783,1733746548634] 2024-12-09T12:15:50,918 DEBUG [RS:0;f15dd41c88ad:38111 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T12:15:50,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T12:15:50,925 DEBUG [RS:0;f15dd41c88ad:38111 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T12:15:50,925 DEBUG [RS:0;f15dd41c88ad:38111 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T12:15:50,925 DEBUG [RS:0;f15dd41c88ad:38111 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f15dd41c88ad,38111,1733746548467 2024-12-09T12:15:50,925 DEBUG [RS:2;f15dd41c88ad:36579 {}] zookeeper.ZKUtil(111): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f15dd41c88ad,36579,1733746548727 2024-12-09T12:15:50,925 DEBUG [RS:0;f15dd41c88ad:38111 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f15dd41c88ad,38111,1733746548467' 2024-12-09T12:15:50,925 DEBUG [RS:0;f15dd41c88ad:38111 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T12:15:50,925 WARN [RS:2;f15dd41c88ad:36579 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T12:15:50,925 INFO [RS:2;f15dd41c88ad:36579 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T12:15:50,925 DEBUG [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,36579,1733746548727 2024-12-09T12:15:50,931 DEBUG [RS:0;f15dd41c88ad:38111 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T12:15:50,932 DEBUG [RS:0;f15dd41c88ad:38111 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T12:15:50,932 INFO [RS:0;f15dd41c88ad:38111 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T12:15:50,932 INFO [RS:0;f15dd41c88ad:38111 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T12:15:50,932 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f15dd41c88ad,36579,1733746548727] 2024-12-09T12:15:50,941 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T12:15:50,942 INFO [RS:1;f15dd41c88ad:33783 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T12:15:50,950 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-09T12:15:50,970 INFO [RS:1;f15dd41c88ad:33783 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T12:15:50,971 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T12:15:50,979 INFO [RS:1;f15dd41c88ad:33783 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T12:15:50,980 INFO [RS:1;f15dd41c88ad:33783 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:50,982 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T12:15:50,983 INFO [RS:1;f15dd41c88ad:33783 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T12:15:50,985 INFO [RS:1;f15dd41c88ad:33783 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T12:15:50,985 INFO [RS:1;f15dd41c88ad:33783 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:50,985 DEBUG [RS:1;f15dd41c88ad:33783 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,985 DEBUG [RS:1;f15dd41c88ad:33783 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,986 DEBUG [RS:1;f15dd41c88ad:33783 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,986 DEBUG [RS:1;f15dd41c88ad:33783 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,986 DEBUG [RS:1;f15dd41c88ad:33783 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,986 DEBUG [RS:1;f15dd41c88ad:33783 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0, corePoolSize=2, maxPoolSize=2 2024-12-09T12:15:50,986 DEBUG [RS:1;f15dd41c88ad:33783 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,986 DEBUG [RS:1;f15dd41c88ad:33783 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,986 DEBUG [RS:1;f15dd41c88ad:33783 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,987 DEBUG [RS:1;f15dd41c88ad:33783 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,987 DEBUG [RS:1;f15dd41c88ad:33783 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,987 DEBUG [RS:1;f15dd41c88ad:33783 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,987 DEBUG [RS:1;f15dd41c88ad:33783 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f15dd41c88ad:0, corePoolSize=3, maxPoolSize=3 2024-12-09T12:15:50,988 INFO [RS:2;f15dd41c88ad:36579 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T12:15:50,988 INFO [RS:2;f15dd41c88ad:36579 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:50,988 DEBUG [RS:1;f15dd41c88ad:33783 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f15dd41c88ad:0, corePoolSize=3, maxPoolSize=3 2024-12-09T12:15:50,989 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T12:15:50,995 INFO [RS:2;f15dd41c88ad:36579 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T12:15:50,995 INFO [RS:2;f15dd41c88ad:36579 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:50,995 DEBUG [RS:2;f15dd41c88ad:36579 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,995 DEBUG [RS:2;f15dd41c88ad:36579 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,996 DEBUG [RS:2;f15dd41c88ad:36579 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,996 DEBUG [RS:2;f15dd41c88ad:36579 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,996 DEBUG [RS:2;f15dd41c88ad:36579 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,996 DEBUG [RS:2;f15dd41c88ad:36579 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0, corePoolSize=2, maxPoolSize=2 2024-12-09T12:15:50,996 DEBUG [RS:2;f15dd41c88ad:36579 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,996 DEBUG [RS:2;f15dd41c88ad:36579 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,997 DEBUG [RS:2;f15dd41c88ad:36579 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,997 DEBUG [RS:2;f15dd41c88ad:36579 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,997 DEBUG [RS:2;f15dd41c88ad:36579 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,997 DEBUG [RS:2;f15dd41c88ad:36579 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f15dd41c88ad:0, corePoolSize=1, maxPoolSize=1 2024-12-09T12:15:50,997 DEBUG [RS:2;f15dd41c88ad:36579 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f15dd41c88ad:0, corePoolSize=3, maxPoolSize=3 2024-12-09T12:15:50,997 DEBUG [RS:2;f15dd41c88ad:36579 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f15dd41c88ad:0, corePoolSize=3, maxPoolSize=3 2024-12-09T12:15:51,000 INFO [RS:1;f15dd41c88ad:33783 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:51,000 INFO [RS:1;f15dd41c88ad:33783 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:51,000 INFO [RS:1;f15dd41c88ad:33783 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:51,000 INFO [RS:1;f15dd41c88ad:33783 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:51,000 INFO [RS:1;f15dd41c88ad:33783 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:51,000 INFO [RS:1;f15dd41c88ad:33783 {}] hbase.ChoreService(168): Chore ScheduledChore name=f15dd41c88ad,33783,1733746548634-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T12:15:51,034 INFO [RS:1;f15dd41c88ad:33783 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T12:15:51,035 INFO [RS:1;f15dd41c88ad:33783 {}] hbase.ChoreService(168): Chore ScheduledChore name=f15dd41c88ad,33783,1733746548634-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:51,035 INFO [RS:1;f15dd41c88ad:33783 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:51,035 INFO [RS:1;f15dd41c88ad:33783 {}] regionserver.Replication(171): f15dd41c88ad,33783,1733746548634 started 2024-12-09T12:15:51,042 INFO [RS:2;f15dd41c88ad:36579 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:51,042 INFO [RS:2;f15dd41c88ad:36579 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:51,042 INFO [RS:2;f15dd41c88ad:36579 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:51,042 INFO [RS:2;f15dd41c88ad:36579 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:51,042 INFO [RS:2;f15dd41c88ad:36579 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:51,042 INFO [RS:2;f15dd41c88ad:36579 {}] hbase.ChoreService(168): Chore ScheduledChore name=f15dd41c88ad,36579,1733746548727-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T12:15:51,045 INFO [RS:0;f15dd41c88ad:38111 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T12:15:51,064 INFO [RS:0;f15dd41c88ad:38111 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f15dd41c88ad%2C38111%2C1733746548467, suffix=, logDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,38111,1733746548467, archiveDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/oldWALs, maxLogs=32 2024-12-09T12:15:51,073 INFO [RS:1;f15dd41c88ad:33783 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:51,085 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T12:15:51,085 INFO [RS:2;f15dd41c88ad:36579 {}] hbase.ChoreService(168): Chore ScheduledChore name=f15dd41c88ad,36579,1733746548727-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:51,086 INFO [RS:2;f15dd41c88ad:36579 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:51,095 INFO [RS:1;f15dd41c88ad:33783 {}] regionserver.HRegionServer(1482): Serving as f15dd41c88ad,33783,1733746548634, RpcServer on f15dd41c88ad/172.17.0.3:33783, sessionid=0x1012b1c64ac0002 2024-12-09T12:15:51,095 DEBUG [RS:1;f15dd41c88ad:33783 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T12:15:51,095 DEBUG [RS:1;f15dd41c88ad:33783 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:51,096 DEBUG [RS:1;f15dd41c88ad:33783 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f15dd41c88ad,33783,1733746548634' 2024-12-09T12:15:51,096 DEBUG [RS:1;f15dd41c88ad:33783 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T12:15:51,096 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.Replication(171): f15dd41c88ad,36579,1733746548727 started 2024-12-09T12:15:51,101 WARN [f15dd41c88ad:42519 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T12:15:51,105 DEBUG [RS:1;f15dd41c88ad:33783 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T12:15:51,107 DEBUG [RS:1;f15dd41c88ad:33783 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T12:15:51,107 DEBUG [RS:1;f15dd41c88ad:33783 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T12:15:51,107 DEBUG [RS:1;f15dd41c88ad:33783 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:51,107 DEBUG [RS:1;f15dd41c88ad:33783 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f15dd41c88ad,33783,1733746548634' 2024-12-09T12:15:51,107 DEBUG [RS:1;f15dd41c88ad:33783 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T12:15:51,113 DEBUG [RS:1;f15dd41c88ad:33783 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T12:15:51,115 DEBUG [RS:1;f15dd41c88ad:33783 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T12:15:51,115 INFO [RS:1;f15dd41c88ad:33783 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T12:15:51,115 INFO [RS:1;f15dd41c88ad:33783 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T12:15:51,121 DEBUG [RS:0;f15dd41c88ad:38111 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,38111,1733746548467/f15dd41c88ad%2C38111%2C1733746548467.1733746551067, exclude list is [], retry=0 2024-12-09T12:15:51,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39373,DS-0456a585-8dc2-4cb5-ae72-660bf02f0e80,DISK] 2024-12-09T12:15:51,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33961,DS-e1bc5ec1-d599-496b-8be4-5493e5661bf9,DISK] 2024-12-09T12:15:51,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33963,DS-be4b45f7-c1d4-4431-b834-2678e63cb272,DISK] 2024-12-09T12:15:51,138 INFO [RS:2;f15dd41c88ad:36579 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:51,139 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(1482): Serving as f15dd41c88ad,36579,1733746548727, RpcServer on f15dd41c88ad/172.17.0.3:36579, sessionid=0x1012b1c64ac0003 2024-12-09T12:15:51,139 DEBUG [RS:2;f15dd41c88ad:36579 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T12:15:51,139 DEBUG [RS:2;f15dd41c88ad:36579 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f15dd41c88ad,36579,1733746548727 2024-12-09T12:15:51,140 DEBUG [RS:2;f15dd41c88ad:36579 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f15dd41c88ad,36579,1733746548727' 2024-12-09T12:15:51,140 DEBUG [RS:2;f15dd41c88ad:36579 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T12:15:51,141 DEBUG [RS:2;f15dd41c88ad:36579 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T12:15:51,142 DEBUG [RS:2;f15dd41c88ad:36579 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T12:15:51,142 DEBUG [RS:2;f15dd41c88ad:36579 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T12:15:51,142 DEBUG [RS:2;f15dd41c88ad:36579 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f15dd41c88ad,36579,1733746548727 2024-12-09T12:15:51,150 DEBUG [RS:2;f15dd41c88ad:36579 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f15dd41c88ad,36579,1733746548727' 2024-12-09T12:15:51,150 DEBUG [RS:2;f15dd41c88ad:36579 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T12:15:51,151 DEBUG [RS:2;f15dd41c88ad:36579 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T12:15:51,153 DEBUG [RS:2;f15dd41c88ad:36579 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T12:15:51,153 INFO [RS:2;f15dd41c88ad:36579 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T12:15:51,153 INFO [RS:2;f15dd41c88ad:36579 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T12:15:51,167 INFO [RS:0;f15dd41c88ad:38111 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,38111,1733746548467/f15dd41c88ad%2C38111%2C1733746548467.1733746551067 2024-12-09T12:15:51,175 DEBUG [RS:0;f15dd41c88ad:38111 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41543:41543),(127.0.0.1/127.0.0.1:42689:42689),(127.0.0.1/127.0.0.1:41475:41475)] 2024-12-09T12:15:51,216 INFO [RS:1;f15dd41c88ad:33783 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T12:15:51,220 INFO [RS:1;f15dd41c88ad:33783 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f15dd41c88ad%2C33783%2C1733746548634, suffix=, logDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634, archiveDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/oldWALs, maxLogs=32 2024-12-09T12:15:51,244 DEBUG [RS:1;f15dd41c88ad:33783 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634/f15dd41c88ad%2C33783%2C1733746548634.1733746551223, exclude list is [], retry=0 2024-12-09T12:15:51,254 INFO [RS:2;f15dd41c88ad:36579 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T12:15:51,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39373,DS-0456a585-8dc2-4cb5-ae72-660bf02f0e80,DISK] 2024-12-09T12:15:51,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33961,DS-e1bc5ec1-d599-496b-8be4-5493e5661bf9,DISK] 2024-12-09T12:15:51,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33963,DS-be4b45f7-c1d4-4431-b834-2678e63cb272,DISK] 2024-12-09T12:15:51,259 INFO [RS:2;f15dd41c88ad:36579 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f15dd41c88ad%2C36579%2C1733746548727, suffix=, logDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,36579,1733746548727, archiveDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/oldWALs, maxLogs=32 2024-12-09T12:15:51,291 INFO [RS:1;f15dd41c88ad:33783 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634/f15dd41c88ad%2C33783%2C1733746548634.1733746551223 2024-12-09T12:15:51,295 DEBUG [RS:2;f15dd41c88ad:36579 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,36579,1733746548727/f15dd41c88ad%2C36579%2C1733746548727.1733746551262, exclude list is [], retry=0 2024-12-09T12:15:51,297 DEBUG [RS:1;f15dd41c88ad:33783 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42689:42689),(127.0.0.1/127.0.0.1:41475:41475),(127.0.0.1/127.0.0.1:41543:41543)] 2024-12-09T12:15:51,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39373,DS-0456a585-8dc2-4cb5-ae72-660bf02f0e80,DISK] 2024-12-09T12:15:51,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33963,DS-be4b45f7-c1d4-4431-b834-2678e63cb272,DISK] 2024-12-09T12:15:51,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33961,DS-e1bc5ec1-d599-496b-8be4-5493e5661bf9,DISK] 2024-12-09T12:15:51,354 DEBUG [f15dd41c88ad:42519 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-09T12:15:51,366 INFO [RS:2;f15dd41c88ad:36579 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,36579,1733746548727/f15dd41c88ad%2C36579%2C1733746548727.1733746551262 2024-12-09T12:15:51,366 DEBUG [f15dd41c88ad:42519 {}] balancer.BalancerClusterState(204): Hosts are {f15dd41c88ad=0} racks are {/default-rack=0} 2024-12-09T12:15:51,372 DEBUG [RS:2;f15dd41c88ad:36579 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42689:42689),(127.0.0.1/127.0.0.1:41475:41475),(127.0.0.1/127.0.0.1:41543:41543)] 2024-12-09T12:15:51,375 DEBUG [f15dd41c88ad:42519 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T12:15:51,375 DEBUG [f15dd41c88ad:42519 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T12:15:51,375 DEBUG [f15dd41c88ad:42519 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T12:15:51,375 DEBUG [f15dd41c88ad:42519 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T12:15:51,376 INFO [f15dd41c88ad:42519 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T12:15:51,376 INFO [f15dd41c88ad:42519 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T12:15:51,376 DEBUG [f15dd41c88ad:42519 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T12:15:51,389 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:51,407 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f15dd41c88ad,33783,1733746548634, state=OPENING 2024-12-09T12:15:51,434 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-09T12:15:51,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T12:15:51,438 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T12:15:51,438 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33783-0x1012b1c64ac0002, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T12:15:51,438 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T12:15:51,439 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:15:51,439 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:15:51,439 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:15:51,440 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:15:51,443 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T12:15:51,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,33783,1733746548634}] 2024-12-09T12:15:51,662 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T12:15:51,682 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55513, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T12:15:51,718 INFO [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-09T12:15:51,719 INFO [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T12:15:51,719 INFO [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-09T12:15:51,727 INFO [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f15dd41c88ad%2C33783%2C1733746548634.meta, suffix=.meta, logDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634, archiveDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/oldWALs, maxLogs=32 2024-12-09T12:15:51,766 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634/f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, exclude list is [], retry=0 2024-12-09T12:15:51,772 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39373,DS-0456a585-8dc2-4cb5-ae72-660bf02f0e80,DISK] 2024-12-09T12:15:51,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33961,DS-e1bc5ec1-d599-496b-8be4-5493e5661bf9,DISK] 2024-12-09T12:15:51,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33963,DS-be4b45f7-c1d4-4431-b834-2678e63cb272,DISK] 2024-12-09T12:15:51,819 INFO [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634/f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta 2024-12-09T12:15:51,827 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42689:42689),(127.0.0.1/127.0.0.1:41475:41475),(127.0.0.1/127.0.0.1:41543:41543)] 2024-12-09T12:15:51,827 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-09T12:15:51,829 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-09T12:15:51,833 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-09T12:15:51,841 INFO [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-09T12:15:51,855 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-09T12:15:51,856 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:51,856 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-09T12:15:51,856 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-09T12:15:51,863 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T12:15:51,865 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T12:15:51,865 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:51,868 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T12:15:51,868 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T12:15:51,870 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T12:15:51,870 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:51,872 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T12:15:51,872 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T12:15:51,879 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T12:15:51,879 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:51,882 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T12:15:51,883 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T12:15:51,885 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T12:15:51,885 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:51,886 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T12:15:51,886 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T12:15:51,891 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740 2024-12-09T12:15:51,896 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740 2024-12-09T12:15:51,900 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T12:15:51,900 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T12:15:51,901 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T12:15:51,908 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T12:15:51,916 INFO [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66417157, jitterRate=-0.01030723750591278}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T12:15:51,917 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-09T12:15:51,919 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733746551857Writing region info on filesystem at 1733746551857Initializing all the Stores at 1733746551861 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733746551861Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733746551862 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746551862Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733746551863 (+1 ms)Cleaning up temporary data from old regions at 1733746551900 (+37 ms)Running coprocessor post-open hooks at 1733746551917 (+17 ms)Region opened successfully at 1733746551918 (+1 ms) 2024-12-09T12:15:51,931 INFO [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733746551650 2024-12-09T12:15:51,947 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-09T12:15:51,947 INFO [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-09T12:15:51,950 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:51,956 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f15dd41c88ad,33783,1733746548634, state=OPEN 2024-12-09T12:15:51,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:15:51,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:15:51,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33783-0x1012b1c64ac0002, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:15:51,963 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:15:51,963 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:15:51,963 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:15:51,963 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:51,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:15:51,971 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:15:51,973 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-09T12:15:51,973 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,33783,1733746548634 in 515 msec 2024-12-09T12:15:51,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-09T12:15:51,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.0620 sec 2024-12-09T12:15:51,992 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T12:15:51,993 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-09T12:15:52,036 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T12:15:52,038 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f15dd41c88ad,33783,1733746548634, seqNum=-1] 2024-12-09T12:15:52,103 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T12:15:52,108 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47437, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T12:15:52,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7060 sec 2024-12-09T12:15:52,158 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733746552158, completionTime=-1 2024-12-09T12:15:52,169 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-09T12:15:52,169 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T12:15:52,234 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-09T12:15:52,234 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733746612234 2024-12-09T12:15:52,234 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733746672234 2024-12-09T12:15:52,234 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 64 msec 2024-12-09T12:15:52,239 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-09T12:15:52,252 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f15dd41c88ad,42519,1733746546940-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:52,252 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f15dd41c88ad,42519,1733746546940-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:52,252 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f15dd41c88ad,42519,1733746546940-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:52,254 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f15dd41c88ad:42519, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:52,255 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:52,261 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:52,267 DEBUG [master/f15dd41c88ad:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-09T12:15:52,318 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.400sec 2024-12-09T12:15:52,340 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-09T12:15:52,342 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-09T12:15:52,343 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-09T12:15:52,344 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-09T12:15:52,344 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-09T12:15:52,345 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f15dd41c88ad,42519,1733746546940-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T12:15:52,346 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f15dd41c88ad,42519,1733746546940-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-09T12:15:52,366 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-09T12:15:52,368 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-09T12:15:52,368 INFO [master/f15dd41c88ad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f15dd41c88ad,42519,1733746546940-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T12:15:52,444 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fd0e425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T12:15:52,451 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-09T12:15:52,451 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-09T12:15:52,459 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f15dd41c88ad,42519,-1 for getting cluster id 2024-12-09T12:15:52,474 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T12:15:52,518 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bd14337c-7e24-4686-a8c1-6c16b6d09ea0' 2024-12-09T12:15:52,531 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T12:15:52,531 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bd14337c-7e24-4686-a8c1-6c16b6d09ea0" 2024-12-09T12:15:52,532 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74fc37d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T12:15:52,533 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f15dd41c88ad,42519,-1] 2024-12-09T12:15:52,536 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T12:15:52,548 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T12:15:52,563 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T12:15:52,572 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c15e346, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T12:15:52,574 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T12:15:52,610 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f15dd41c88ad,33783,1733746548634, seqNum=-1] 2024-12-09T12:15:52,611 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T12:15:52,642 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T12:15:52,795 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f15dd41c88ad,42519,1733746546940 2024-12-09T12:15:52,805 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T12:15:52,833 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is f15dd41c88ad,42519,1733746546940 2024-12-09T12:15:52,846 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7b3a1243 2024-12-09T12:15:52,847 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T12:15:52,864 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T12:15:52,883 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T12:15:52,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-12-09T12:15:52,911 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T12:15:52,914 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:52,927 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T12:15:52,930 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-12-09T12:15:52,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T12:15:53,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741837_1013 (size=358) 2024-12-09T12:15:53,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741837_1013 (size=358) 2024-12-09T12:15:53,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741837_1013 (size=358) 2024-12-09T12:15:53,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T12:15:53,068 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cc7aaad28378d11daf90725c0f9d39f5, NAME => 'test,,1733746552866.cc7aaad28378d11daf90725c0f9d39f5.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:53,078 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 9ac6650f2a35b2c131e93ead260da106, NAME => 'test,bbb,1733746552866.9ac6650f2a35b2c131e93ead260da106.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:53,080 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 8bc29ffded7f1b74a1ee2986c5e611f8, NAME => 'test,ccc,1733746552866.8bc29ffded7f1b74a1ee2986c5e611f8.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:53,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741839_1015 (size=45) 2024-12-09T12:15:53,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741839_1015 (size=45) 2024-12-09T12:15:53,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741839_1015 (size=45) 2024-12-09T12:15:53,235 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1733746552866.8bc29ffded7f1b74a1ee2986c5e611f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:53,236 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 8bc29ffded7f1b74a1ee2986c5e611f8, disabling compactions & flushes 2024-12-09T12:15:53,236 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1733746552866.8bc29ffded7f1b74a1ee2986c5e611f8. 2024-12-09T12:15:53,237 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1733746552866.8bc29ffded7f1b74a1ee2986c5e611f8. 2024-12-09T12:15:53,237 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1733746552866.8bc29ffded7f1b74a1ee2986c5e611f8. after waiting 0 ms 2024-12-09T12:15:53,237 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1733746552866.8bc29ffded7f1b74a1ee2986c5e611f8. 2024-12-09T12:15:53,237 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1733746552866.8bc29ffded7f1b74a1ee2986c5e611f8. 2024-12-09T12:15:53,237 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 8bc29ffded7f1b74a1ee2986c5e611f8: Waiting for close lock at 1733746553236Disabling compacts and flushes for region at 1733746553236Disabling writes for close at 1733746553237 (+1 ms)Writing region close event to WAL at 1733746553237Closed at 1733746553237 2024-12-09T12:15:53,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741838_1014 (size=42) 2024-12-09T12:15:53,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741840_1016 (size=45) 2024-12-09T12:15:53,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741840_1016 (size=45) 2024-12-09T12:15:53,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741840_1016 (size=45) 2024-12-09T12:15:53,244 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1733746552866.9ac6650f2a35b2c131e93ead260da106.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:53,244 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 9ac6650f2a35b2c131e93ead260da106, disabling compactions & flushes 2024-12-09T12:15:53,244 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1733746552866.9ac6650f2a35b2c131e93ead260da106. 2024-12-09T12:15:53,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741838_1014 (size=42) 2024-12-09T12:15:53,244 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1733746552866.9ac6650f2a35b2c131e93ead260da106. 2024-12-09T12:15:53,244 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1733746552866.9ac6650f2a35b2c131e93ead260da106. after waiting 0 ms 2024-12-09T12:15:53,244 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1733746552866.9ac6650f2a35b2c131e93ead260da106. 2024-12-09T12:15:53,244 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1733746552866.9ac6650f2a35b2c131e93ead260da106. 2024-12-09T12:15:53,245 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 9ac6650f2a35b2c131e93ead260da106: Waiting for close lock at 1733746553244Disabling compacts and flushes for region at 1733746553244Disabling writes for close at 1733746553244Writing region close event to WAL at 1733746553244Closed at 1733746553244 2024-12-09T12:15:53,245 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => e5251eee3971dde9f0f276cfd8632c21, NAME => 'test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:53,256 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 1ce28e8d0ce113d2637ab328f5ffd3be, NAME => 'test,ddd,1733746552866.1ce28e8d0ce113d2637ab328f5ffd3be.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:53,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741838_1014 (size=42) 2024-12-09T12:15:53,271 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1733746552866.cc7aaad28378d11daf90725c0f9d39f5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:53,271 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing cc7aaad28378d11daf90725c0f9d39f5, disabling compactions & flushes 2024-12-09T12:15:53,271 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1733746552866.cc7aaad28378d11daf90725c0f9d39f5. 2024-12-09T12:15:53,271 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1733746552866.cc7aaad28378d11daf90725c0f9d39f5. 2024-12-09T12:15:53,271 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1733746552866.cc7aaad28378d11daf90725c0f9d39f5. after waiting 0 ms 2024-12-09T12:15:53,271 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1733746552866.cc7aaad28378d11daf90725c0f9d39f5. 2024-12-09T12:15:53,271 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1733746552866.cc7aaad28378d11daf90725c0f9d39f5. 2024-12-09T12:15:53,271 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for cc7aaad28378d11daf90725c0f9d39f5: Waiting for close lock at 1733746553271Disabling compacts and flushes for region at 1733746553271Disabling writes for close at 1733746553271Writing region close event to WAL at 1733746553271Closed at 1733746553271 2024-12-09T12:15:53,272 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6b5306e854c5aec62eff62a18cec5859, NAME => 'test,fff,1733746552866.6b5306e854c5aec62eff62a18cec5859.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:53,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T12:15:53,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741841_1017 (size=45) 2024-12-09T12:15:53,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741841_1017 (size=45) 2024-12-09T12:15:53,458 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:53,458 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing e5251eee3971dde9f0f276cfd8632c21, disabling compactions & flushes 2024-12-09T12:15:53,458 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21. 2024-12-09T12:15:53,458 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21. 2024-12-09T12:15:53,458 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21. after waiting 0 ms 2024-12-09T12:15:53,458 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21. 2024-12-09T12:15:53,458 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21. 2024-12-09T12:15:53,458 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for e5251eee3971dde9f0f276cfd8632c21: Waiting for close lock at 1733746553458Disabling compacts and flushes for region at 1733746553458Disabling writes for close at 1733746553458Writing region close event to WAL at 1733746553458Closed at 1733746553458 2024-12-09T12:15:53,459 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => c910d27fbf3265165c7fd59609e57c82, NAME => 'test,ggg,1733746552866.c910d27fbf3265165c7fd59609e57c82.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:53,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741841_1017 (size=45) 2024-12-09T12:15:53,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741842_1018 (size=45) 2024-12-09T12:15:53,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741842_1018 (size=45) 2024-12-09T12:15:53,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741842_1018 (size=45) 2024-12-09T12:15:53,464 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ddd,1733746552866.1ce28e8d0ce113d2637ab328f5ffd3be.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:53,465 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 1ce28e8d0ce113d2637ab328f5ffd3be, disabling compactions & flushes 2024-12-09T12:15:53,465 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ddd,1733746552866.1ce28e8d0ce113d2637ab328f5ffd3be. 2024-12-09T12:15:53,465 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1733746552866.1ce28e8d0ce113d2637ab328f5ffd3be. 2024-12-09T12:15:53,465 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1733746552866.1ce28e8d0ce113d2637ab328f5ffd3be. after waiting 0 ms 2024-12-09T12:15:53,465 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1733746552866.1ce28e8d0ce113d2637ab328f5ffd3be. 2024-12-09T12:15:53,465 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ddd,1733746552866.1ce28e8d0ce113d2637ab328f5ffd3be. 2024-12-09T12:15:53,465 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 1ce28e8d0ce113d2637ab328f5ffd3be: Waiting for close lock at 1733746553464Disabling compacts and flushes for region at 1733746553464Disabling writes for close at 1733746553465 (+1 ms)Writing region close event to WAL at 1733746553465Closed at 1733746553465 2024-12-09T12:15:53,466 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => a258d2ecd9a494e8d56406b774da28ef, NAME => 'test,hhh,1733746552866.a258d2ecd9a494e8d56406b774da28ef.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:53,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741843_1019 (size=45) 2024-12-09T12:15:53,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741843_1019 (size=45) 2024-12-09T12:15:53,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741843_1019 (size=45) 2024-12-09T12:15:53,484 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,fff,1733746552866.6b5306e854c5aec62eff62a18cec5859.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:53,485 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 6b5306e854c5aec62eff62a18cec5859, disabling compactions & flushes 2024-12-09T12:15:53,485 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,fff,1733746552866.6b5306e854c5aec62eff62a18cec5859. 2024-12-09T12:15:53,485 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1733746552866.6b5306e854c5aec62eff62a18cec5859. 2024-12-09T12:15:53,485 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1733746552866.6b5306e854c5aec62eff62a18cec5859. after waiting 0 ms 2024-12-09T12:15:53,485 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1733746552866.6b5306e854c5aec62eff62a18cec5859. 2024-12-09T12:15:53,485 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,fff,1733746552866.6b5306e854c5aec62eff62a18cec5859. 2024-12-09T12:15:53,485 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6b5306e854c5aec62eff62a18cec5859: Waiting for close lock at 1733746553485Disabling compacts and flushes for region at 1733746553485Disabling writes for close at 1733746553485Writing region close event to WAL at 1733746553485Closed at 1733746553485 2024-12-09T12:15:53,486 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a0cf25c9917186a9e3dbd87e44f33615, NAME => 'test,iii,1733746552866.a0cf25c9917186a9e3dbd87e44f33615.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:53,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741845_1021 (size=45) 2024-12-09T12:15:53,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741845_1021 (size=45) 2024-12-09T12:15:53,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741845_1021 (size=45) 2024-12-09T12:15:53,581 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,hhh,1733746552866.a258d2ecd9a494e8d56406b774da28ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:53,581 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing a258d2ecd9a494e8d56406b774da28ef, disabling compactions & flushes 2024-12-09T12:15:53,582 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,hhh,1733746552866.a258d2ecd9a494e8d56406b774da28ef. 2024-12-09T12:15:53,582 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1733746552866.a258d2ecd9a494e8d56406b774da28ef. 2024-12-09T12:15:53,582 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1733746552866.a258d2ecd9a494e8d56406b774da28ef. after waiting 0 ms 2024-12-09T12:15:53,582 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1733746552866.a258d2ecd9a494e8d56406b774da28ef. 2024-12-09T12:15:53,582 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,hhh,1733746552866.a258d2ecd9a494e8d56406b774da28ef. 2024-12-09T12:15:53,582 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for a258d2ecd9a494e8d56406b774da28ef: Waiting for close lock at 1733746553581Disabling compacts and flushes for region at 1733746553581Disabling writes for close at 1733746553582 (+1 ms)Writing region close event to WAL at 1733746553582Closed at 1733746553582 2024-12-09T12:15:53,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T12:15:53,583 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => d9e23742363329565223433d05d86296, NAME => 'test,jjj,1733746552866.d9e23742363329565223433d05d86296.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:53,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741844_1020 (size=45) 2024-12-09T12:15:53,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741844_1020 (size=45) 2024-12-09T12:15:53,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741844_1020 (size=45) 2024-12-09T12:15:53,599 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ggg,1733746552866.c910d27fbf3265165c7fd59609e57c82.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:53,599 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing c910d27fbf3265165c7fd59609e57c82, disabling compactions & flushes 2024-12-09T12:15:53,599 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ggg,1733746552866.c910d27fbf3265165c7fd59609e57c82. 2024-12-09T12:15:53,599 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1733746552866.c910d27fbf3265165c7fd59609e57c82. 2024-12-09T12:15:53,599 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1733746552866.c910d27fbf3265165c7fd59609e57c82. after waiting 0 ms 2024-12-09T12:15:53,599 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1733746552866.c910d27fbf3265165c7fd59609e57c82. 2024-12-09T12:15:53,599 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ggg,1733746552866.c910d27fbf3265165c7fd59609e57c82. 2024-12-09T12:15:53,599 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for c910d27fbf3265165c7fd59609e57c82: Waiting for close lock at 1733746553599Disabling compacts and flushes for region at 1733746553599Disabling writes for close at 1733746553599Writing region close event to WAL at 1733746553599Closed at 1733746553599 2024-12-09T12:15:53,600 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => c9702c569de70ef36018a26943c3226f, NAME => 'test,kkk,1733746552866.c9702c569de70ef36018a26943c3226f.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:53,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741846_1022 (size=45) 2024-12-09T12:15:53,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741846_1022 (size=45) 2024-12-09T12:15:53,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741846_1022 (size=45) 2024-12-09T12:15:53,607 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,iii,1733746552866.a0cf25c9917186a9e3dbd87e44f33615.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:53,607 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing a0cf25c9917186a9e3dbd87e44f33615, disabling compactions & flushes 2024-12-09T12:15:53,607 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,iii,1733746552866.a0cf25c9917186a9e3dbd87e44f33615. 2024-12-09T12:15:53,607 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1733746552866.a0cf25c9917186a9e3dbd87e44f33615. 2024-12-09T12:15:53,607 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1733746552866.a0cf25c9917186a9e3dbd87e44f33615. after waiting 0 ms 2024-12-09T12:15:53,607 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1733746552866.a0cf25c9917186a9e3dbd87e44f33615. 2024-12-09T12:15:53,607 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,iii,1733746552866.a0cf25c9917186a9e3dbd87e44f33615. 2024-12-09T12:15:53,607 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for a0cf25c9917186a9e3dbd87e44f33615: Waiting for close lock at 1733746553607Disabling compacts and flushes for region at 1733746553607Disabling writes for close at 1733746553607Writing region close event to WAL at 1733746553607Closed at 1733746553607 2024-12-09T12:15:53,608 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ebee1500f62faadfac8720740a1e527e, NAME => 'test,lll,1733746552866.ebee1500f62faadfac8720740a1e527e.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:53,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741849_1025 (size=45) 2024-12-09T12:15:53,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741848_1024 (size=45) 2024-12-09T12:15:53,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741848_1024 (size=45) 2024-12-09T12:15:53,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741849_1025 (size=45) 2024-12-09T12:15:53,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741849_1025 (size=45) 2024-12-09T12:15:53,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741848_1024 (size=45) 2024-12-09T12:15:53,708 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,lll,1733746552866.ebee1500f62faadfac8720740a1e527e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:53,708 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing ebee1500f62faadfac8720740a1e527e, disabling compactions & flushes 2024-12-09T12:15:53,708 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,lll,1733746552866.ebee1500f62faadfac8720740a1e527e. 2024-12-09T12:15:53,708 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1733746552866.ebee1500f62faadfac8720740a1e527e. 2024-12-09T12:15:53,708 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1733746552866.ebee1500f62faadfac8720740a1e527e. after waiting 0 ms 2024-12-09T12:15:53,708 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1733746552866.ebee1500f62faadfac8720740a1e527e. 2024-12-09T12:15:53,708 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,lll,1733746552866.ebee1500f62faadfac8720740a1e527e. 2024-12-09T12:15:53,708 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for ebee1500f62faadfac8720740a1e527e: Waiting for close lock at 1733746553708Disabling compacts and flushes for region at 1733746553708Disabling writes for close at 1733746553708Writing region close event to WAL at 1733746553708Closed at 1733746553708 2024-12-09T12:15:53,709 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 62a60a0dff0c026a110e755e6c249001, NAME => 'test,mmm,1733746552866.62a60a0dff0c026a110e755e6c249001.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:53,718 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,kkk,1733746552866.c9702c569de70ef36018a26943c3226f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:53,718 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing c9702c569de70ef36018a26943c3226f, disabling compactions & flushes 2024-12-09T12:15:53,718 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,kkk,1733746552866.c9702c569de70ef36018a26943c3226f. 2024-12-09T12:15:53,718 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1733746552866.c9702c569de70ef36018a26943c3226f. 2024-12-09T12:15:53,718 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1733746552866.c9702c569de70ef36018a26943c3226f. after waiting 0 ms 2024-12-09T12:15:53,718 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1733746552866.c9702c569de70ef36018a26943c3226f. 2024-12-09T12:15:53,719 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,kkk,1733746552866.c9702c569de70ef36018a26943c3226f. 2024-12-09T12:15:53,719 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for c9702c569de70ef36018a26943c3226f: Waiting for close lock at 1733746553718Disabling compacts and flushes for region at 1733746553718Disabling writes for close at 1733746553718Writing region close event to WAL at 1733746553718Closed at 1733746553718 2024-12-09T12:15:53,720 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => fdcb3d892a0c9cf641eaedbbd9ec5937, NAME => 'test,nnn,1733746552866.fdcb3d892a0c9cf641eaedbbd9ec5937.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:53,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741847_1023 (size=45) 2024-12-09T12:15:53,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741847_1023 (size=45) 2024-12-09T12:15:53,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741847_1023 (size=45) 2024-12-09T12:15:53,763 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,jjj,1733746552866.d9e23742363329565223433d05d86296.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:53,763 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing d9e23742363329565223433d05d86296, disabling compactions & flushes 2024-12-09T12:15:53,763 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,jjj,1733746552866.d9e23742363329565223433d05d86296. 2024-12-09T12:15:53,764 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1733746552866.d9e23742363329565223433d05d86296. 2024-12-09T12:15:53,764 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1733746552866.d9e23742363329565223433d05d86296. after waiting 0 ms 2024-12-09T12:15:53,764 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1733746552866.d9e23742363329565223433d05d86296. 2024-12-09T12:15:53,764 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,jjj,1733746552866.d9e23742363329565223433d05d86296. 2024-12-09T12:15:53,764 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for d9e23742363329565223433d05d86296: Waiting for close lock at 1733746553763Disabling compacts and flushes for region at 1733746553763Disabling writes for close at 1733746553764 (+1 ms)Writing region close event to WAL at 1733746553764Closed at 1733746553764 2024-12-09T12:15:53,765 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 3a1205a417c23329b514f783f4727022, NAME => 'test,ooo,1733746552866.3a1205a417c23329b514f783f4727022.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:53,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741851_1027 (size=45) 2024-12-09T12:15:53,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741851_1027 (size=45) 2024-12-09T12:15:53,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741851_1027 (size=45) 2024-12-09T12:15:53,903 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,mmm,1733746552866.62a60a0dff0c026a110e755e6c249001.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:53,923 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 62a60a0dff0c026a110e755e6c249001, disabling compactions & flushes 2024-12-09T12:15:53,923 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,mmm,1733746552866.62a60a0dff0c026a110e755e6c249001. 2024-12-09T12:15:53,923 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1733746552866.62a60a0dff0c026a110e755e6c249001. 2024-12-09T12:15:53,923 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1733746552866.62a60a0dff0c026a110e755e6c249001. after waiting 0 ms 2024-12-09T12:15:53,923 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1733746552866.62a60a0dff0c026a110e755e6c249001. 2024-12-09T12:15:53,923 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,mmm,1733746552866.62a60a0dff0c026a110e755e6c249001. 2024-12-09T12:15:53,923 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 62a60a0dff0c026a110e755e6c249001: Waiting for close lock at 1733746553922Disabling compacts and flushes for region at 1733746553922Disabling writes for close at 1733746553923 (+1 ms)Writing region close event to WAL at 1733746553923Closed at 1733746553923 2024-12-09T12:15:53,924 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 43f38f304815b66d00adc6292e217f1a, NAME => 'test,ppp,1733746552866.43f38f304815b66d00adc6292e217f1a.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:53,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741850_1026 (size=45) 2024-12-09T12:15:53,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741850_1026 (size=45) 2024-12-09T12:15:53,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741850_1026 (size=45) 2024-12-09T12:15:53,944 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,nnn,1733746552866.fdcb3d892a0c9cf641eaedbbd9ec5937.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:53,944 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing fdcb3d892a0c9cf641eaedbbd9ec5937, disabling compactions & flushes 2024-12-09T12:15:53,944 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,nnn,1733746552866.fdcb3d892a0c9cf641eaedbbd9ec5937. 2024-12-09T12:15:53,944 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1733746552866.fdcb3d892a0c9cf641eaedbbd9ec5937. 2024-12-09T12:15:53,944 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1733746552866.fdcb3d892a0c9cf641eaedbbd9ec5937. after waiting 0 ms 2024-12-09T12:15:53,944 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1733746552866.fdcb3d892a0c9cf641eaedbbd9ec5937. 2024-12-09T12:15:53,944 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,nnn,1733746552866.fdcb3d892a0c9cf641eaedbbd9ec5937. 2024-12-09T12:15:53,944 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for fdcb3d892a0c9cf641eaedbbd9ec5937: Waiting for close lock at 1733746553944Disabling compacts and flushes for region at 1733746553944Disabling writes for close at 1733746553944Writing region close event to WAL at 1733746553944Closed at 1733746553944 2024-12-09T12:15:53,945 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => fab7cda743d5cc81945ffb7c431d6346, NAME => 'test,qqq,1733746552866.fab7cda743d5cc81945ffb7c431d6346.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:53,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741852_1028 (size=45) 2024-12-09T12:15:53,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741852_1028 (size=45) 2024-12-09T12:15:53,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741852_1028 (size=45) 2024-12-09T12:15:53,993 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ooo,1733746552866.3a1205a417c23329b514f783f4727022.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:53,994 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 3a1205a417c23329b514f783f4727022, disabling compactions & flushes 2024-12-09T12:15:53,994 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ooo,1733746552866.3a1205a417c23329b514f783f4727022. 2024-12-09T12:15:53,994 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1733746552866.3a1205a417c23329b514f783f4727022. 2024-12-09T12:15:53,994 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1733746552866.3a1205a417c23329b514f783f4727022. after waiting 0 ms 2024-12-09T12:15:53,994 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1733746552866.3a1205a417c23329b514f783f4727022. 2024-12-09T12:15:53,994 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ooo,1733746552866.3a1205a417c23329b514f783f4727022. 2024-12-09T12:15:53,994 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 3a1205a417c23329b514f783f4727022: Waiting for close lock at 1733746553993Disabling compacts and flushes for region at 1733746553993Disabling writes for close at 1733746553994 (+1 ms)Writing region close event to WAL at 1733746553994Closed at 1733746553994 2024-12-09T12:15:53,997 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => e0adbf1ba01f94cad79f2814d407182f, NAME => 'test,rrr,1733746552866.e0adbf1ba01f94cad79f2814d407182f.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:54,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741853_1029 (size=45) 2024-12-09T12:15:54,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741853_1029 (size=45) 2024-12-09T12:15:54,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741853_1029 (size=45) 2024-12-09T12:15:54,092 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ppp,1733746552866.43f38f304815b66d00adc6292e217f1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:54,092 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 43f38f304815b66d00adc6292e217f1a, disabling compactions & flushes 2024-12-09T12:15:54,092 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ppp,1733746552866.43f38f304815b66d00adc6292e217f1a. 2024-12-09T12:15:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T12:15:54,106 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1733746552866.43f38f304815b66d00adc6292e217f1a. 2024-12-09T12:15:54,107 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1733746552866.43f38f304815b66d00adc6292e217f1a. after waiting 0 ms 2024-12-09T12:15:54,107 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1733746552866.43f38f304815b66d00adc6292e217f1a. 2024-12-09T12:15:54,107 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ppp,1733746552866.43f38f304815b66d00adc6292e217f1a. 2024-12-09T12:15:54,107 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 43f38f304815b66d00adc6292e217f1a: Waiting for close lock at 1733746554092Disabling compacts and flushes for region at 1733746554092Disabling writes for close at 1733746554107 (+15 ms)Writing region close event to WAL at 1733746554107Closed at 1733746554107 2024-12-09T12:15:54,108 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 603cf0cbb0e068a5f7727e4abab3a5d5, NAME => 'test,sss,1733746552866.603cf0cbb0e068a5f7727e4abab3a5d5.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:54,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741854_1030 (size=45) 2024-12-09T12:15:54,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741854_1030 (size=45) 2024-12-09T12:15:54,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741854_1030 (size=45) 2024-12-09T12:15:54,139 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,qqq,1733746552866.fab7cda743d5cc81945ffb7c431d6346.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:54,139 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing fab7cda743d5cc81945ffb7c431d6346, disabling compactions & flushes 2024-12-09T12:15:54,139 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,qqq,1733746552866.fab7cda743d5cc81945ffb7c431d6346. 2024-12-09T12:15:54,139 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1733746552866.fab7cda743d5cc81945ffb7c431d6346. 2024-12-09T12:15:54,139 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1733746552866.fab7cda743d5cc81945ffb7c431d6346. after waiting 0 ms 2024-12-09T12:15:54,139 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1733746552866.fab7cda743d5cc81945ffb7c431d6346. 2024-12-09T12:15:54,139 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,qqq,1733746552866.fab7cda743d5cc81945ffb7c431d6346. 2024-12-09T12:15:54,140 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for fab7cda743d5cc81945ffb7c431d6346: Waiting for close lock at 1733746554139Disabling compacts and flushes for region at 1733746554139Disabling writes for close at 1733746554139Writing region close event to WAL at 1733746554139Closed at 1733746554139 2024-12-09T12:15:54,140 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => ba4a26e129719340e5696ac56c6b61ac, NAME => 'test,ttt,1733746552866.ba4a26e129719340e5696ac56c6b61ac.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:54,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741855_1031 (size=45) 2024-12-09T12:15:54,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741855_1031 (size=45) 2024-12-09T12:15:54,153 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,rrr,1733746552866.e0adbf1ba01f94cad79f2814d407182f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:54,153 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing e0adbf1ba01f94cad79f2814d407182f, disabling compactions & flushes 2024-12-09T12:15:54,154 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,rrr,1733746552866.e0adbf1ba01f94cad79f2814d407182f. 2024-12-09T12:15:54,154 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1733746552866.e0adbf1ba01f94cad79f2814d407182f. 2024-12-09T12:15:54,154 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1733746552866.e0adbf1ba01f94cad79f2814d407182f. after waiting 0 ms 2024-12-09T12:15:54,154 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1733746552866.e0adbf1ba01f94cad79f2814d407182f. 2024-12-09T12:15:54,154 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,rrr,1733746552866.e0adbf1ba01f94cad79f2814d407182f. 2024-12-09T12:15:54,154 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for e0adbf1ba01f94cad79f2814d407182f: Waiting for close lock at 1733746554153Disabling compacts and flushes for region at 1733746554153Disabling writes for close at 1733746554154 (+1 ms)Writing region close event to WAL at 1733746554154Closed at 1733746554154 2024-12-09T12:15:54,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741855_1031 (size=45) 2024-12-09T12:15:54,155 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => b0351c7e96a16dae18ff5a8d1664b2b1, NAME => 'test,uuu,1733746552866.b0351c7e96a16dae18ff5a8d1664b2b1.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:54,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741856_1032 (size=45) 2024-12-09T12:15:54,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741856_1032 (size=45) 2024-12-09T12:15:54,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741856_1032 (size=45) 2024-12-09T12:15:54,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741857_1033 (size=45) 2024-12-09T12:15:54,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741857_1033 (size=45) 2024-12-09T12:15:54,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741857_1033 (size=45) 2024-12-09T12:15:54,270 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ttt,1733746552866.ba4a26e129719340e5696ac56c6b61ac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:54,270 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing ba4a26e129719340e5696ac56c6b61ac, disabling compactions & flushes 2024-12-09T12:15:54,271 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ttt,1733746552866.ba4a26e129719340e5696ac56c6b61ac. 2024-12-09T12:15:54,271 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1733746552866.ba4a26e129719340e5696ac56c6b61ac. 2024-12-09T12:15:54,271 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1733746552866.ba4a26e129719340e5696ac56c6b61ac. after waiting 0 ms 2024-12-09T12:15:54,271 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1733746552866.ba4a26e129719340e5696ac56c6b61ac. 2024-12-09T12:15:54,271 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ttt,1733746552866.ba4a26e129719340e5696ac56c6b61ac. 2024-12-09T12:15:54,271 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for ba4a26e129719340e5696ac56c6b61ac: Waiting for close lock at 1733746554270Disabling compacts and flushes for region at 1733746554270Disabling writes for close at 1733746554271 (+1 ms)Writing region close event to WAL at 1733746554271Closed at 1733746554271 2024-12-09T12:15:54,272 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 23280f1743db631eaa35bf3487e69147, NAME => 'test,vvv,1733746552866.23280f1743db631eaa35bf3487e69147.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:54,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741858_1034 (size=45) 2024-12-09T12:15:54,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741858_1034 (size=45) 2024-12-09T12:15:54,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741858_1034 (size=45) 2024-12-09T12:15:54,326 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,uuu,1733746552866.b0351c7e96a16dae18ff5a8d1664b2b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:54,327 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing b0351c7e96a16dae18ff5a8d1664b2b1, disabling compactions & flushes 2024-12-09T12:15:54,327 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,uuu,1733746552866.b0351c7e96a16dae18ff5a8d1664b2b1. 2024-12-09T12:15:54,327 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1733746552866.b0351c7e96a16dae18ff5a8d1664b2b1. 2024-12-09T12:15:54,327 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1733746552866.b0351c7e96a16dae18ff5a8d1664b2b1. after waiting 0 ms 2024-12-09T12:15:54,327 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1733746552866.b0351c7e96a16dae18ff5a8d1664b2b1. 2024-12-09T12:15:54,327 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,uuu,1733746552866.b0351c7e96a16dae18ff5a8d1664b2b1. 2024-12-09T12:15:54,327 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for b0351c7e96a16dae18ff5a8d1664b2b1: Waiting for close lock at 1733746554326Disabling compacts and flushes for region at 1733746554326Disabling writes for close at 1733746554327 (+1 ms)Writing region close event to WAL at 1733746554327Closed at 1733746554327 2024-12-09T12:15:54,328 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 8ceebab6d8b44ccd74ad65e871405953, NAME => 'test,www,1733746552866.8ceebab6d8b44ccd74ad65e871405953.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:54,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741860_1036 (size=45) 2024-12-09T12:15:54,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741860_1036 (size=45) 2024-12-09T12:15:54,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741860_1036 (size=45) 2024-12-09T12:15:54,433 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,vvv,1733746552866.23280f1743db631eaa35bf3487e69147.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:54,434 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 23280f1743db631eaa35bf3487e69147, disabling compactions & flushes 2024-12-09T12:15:54,434 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,vvv,1733746552866.23280f1743db631eaa35bf3487e69147. 2024-12-09T12:15:54,434 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1733746552866.23280f1743db631eaa35bf3487e69147. 2024-12-09T12:15:54,434 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1733746552866.23280f1743db631eaa35bf3487e69147. after waiting 0 ms 2024-12-09T12:15:54,434 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1733746552866.23280f1743db631eaa35bf3487e69147. 2024-12-09T12:15:54,434 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,vvv,1733746552866.23280f1743db631eaa35bf3487e69147. 2024-12-09T12:15:54,434 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 23280f1743db631eaa35bf3487e69147: Waiting for close lock at 1733746554433Disabling compacts and flushes for region at 1733746554434 (+1 ms)Disabling writes for close at 1733746554434Writing region close event to WAL at 1733746554434Closed at 1733746554434 2024-12-09T12:15:54,435 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 6a763ac99f0e3a37f94a31b5f0479530, NAME => 'test,xxx,1733746552866.6a763ac99f0e3a37f94a31b5f0479530.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:54,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741859_1035 (size=45) 2024-12-09T12:15:54,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741859_1035 (size=45) 2024-12-09T12:15:54,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741859_1035 (size=45) 2024-12-09T12:15:54,446 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,www,1733746552866.8ceebab6d8b44ccd74ad65e871405953.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:54,446 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 8ceebab6d8b44ccd74ad65e871405953, disabling compactions & flushes 2024-12-09T12:15:54,446 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,www,1733746552866.8ceebab6d8b44ccd74ad65e871405953. 2024-12-09T12:15:54,446 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1733746552866.8ceebab6d8b44ccd74ad65e871405953. 2024-12-09T12:15:54,446 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1733746552866.8ceebab6d8b44ccd74ad65e871405953. after waiting 0 ms 2024-12-09T12:15:54,446 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1733746552866.8ceebab6d8b44ccd74ad65e871405953. 2024-12-09T12:15:54,446 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,www,1733746552866.8ceebab6d8b44ccd74ad65e871405953. 2024-12-09T12:15:54,446 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 8ceebab6d8b44ccd74ad65e871405953: Waiting for close lock at 1733746554446Disabling compacts and flushes for region at 1733746554446Disabling writes for close at 1733746554446Writing region close event to WAL at 1733746554446Closed at 1733746554446 2024-12-09T12:15:54,447 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 7eaca6c428d8bff0cd952b3c2dfe5408, NAME => 'test,yyy,1733746552866.7eaca6c428d8bff0cd952b3c2dfe5408.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:54,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741861_1037 (size=45) 2024-12-09T12:15:54,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741861_1037 (size=45) 2024-12-09T12:15:54,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741861_1037 (size=45) 2024-12-09T12:15:54,527 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,xxx,1733746552866.6a763ac99f0e3a37f94a31b5f0479530.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:54,527 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 6a763ac99f0e3a37f94a31b5f0479530, disabling compactions & flushes 2024-12-09T12:15:54,527 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,xxx,1733746552866.6a763ac99f0e3a37f94a31b5f0479530. 2024-12-09T12:15:54,527 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1733746552866.6a763ac99f0e3a37f94a31b5f0479530. 2024-12-09T12:15:54,528 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1733746552866.6a763ac99f0e3a37f94a31b5f0479530. after waiting 0 ms 2024-12-09T12:15:54,528 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1733746552866.6a763ac99f0e3a37f94a31b5f0479530. 2024-12-09T12:15:54,528 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,xxx,1733746552866.6a763ac99f0e3a37f94a31b5f0479530. 2024-12-09T12:15:54,528 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 6a763ac99f0e3a37f94a31b5f0479530: Waiting for close lock at 1733746554527Disabling compacts and flushes for region at 1733746554527Disabling writes for close at 1733746554528 (+1 ms)Writing region close event to WAL at 1733746554528Closed at 1733746554528 2024-12-09T12:15:54,529 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => c8f8bc12489e5c0370ba59c751783a23, NAME => 'test,zzz,1733746552866.c8f8bc12489e5c0370ba59c751783a23.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee 2024-12-09T12:15:54,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741862_1038 (size=45) 2024-12-09T12:15:54,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741862_1038 (size=45) 2024-12-09T12:15:54,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741862_1038 (size=45) 2024-12-09T12:15:54,556 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,yyy,1733746552866.7eaca6c428d8bff0cd952b3c2dfe5408.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:54,556 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 7eaca6c428d8bff0cd952b3c2dfe5408, disabling compactions & flushes 2024-12-09T12:15:54,556 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,yyy,1733746552866.7eaca6c428d8bff0cd952b3c2dfe5408. 2024-12-09T12:15:54,556 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1733746552866.7eaca6c428d8bff0cd952b3c2dfe5408. 2024-12-09T12:15:54,556 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1733746552866.7eaca6c428d8bff0cd952b3c2dfe5408. after waiting 0 ms 2024-12-09T12:15:54,556 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1733746552866.7eaca6c428d8bff0cd952b3c2dfe5408. 2024-12-09T12:15:54,556 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,yyy,1733746552866.7eaca6c428d8bff0cd952b3c2dfe5408. 2024-12-09T12:15:54,556 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 7eaca6c428d8bff0cd952b3c2dfe5408: Waiting for close lock at 1733746554556Disabling compacts and flushes for region at 1733746554556Disabling writes for close at 1733746554556Writing region close event to WAL at 1733746554556Closed at 1733746554556 2024-12-09T12:15:54,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741863_1039 (size=42) 2024-12-09T12:15:54,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741863_1039 (size=42) 2024-12-09T12:15:54,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741863_1039 (size=42) 2024-12-09T12:15:54,588 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,zzz,1733746552866.c8f8bc12489e5c0370ba59c751783a23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:54,588 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing c8f8bc12489e5c0370ba59c751783a23, disabling compactions & flushes 2024-12-09T12:15:54,588 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,zzz,1733746552866.c8f8bc12489e5c0370ba59c751783a23. 2024-12-09T12:15:54,588 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1733746552866.c8f8bc12489e5c0370ba59c751783a23. 2024-12-09T12:15:54,588 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1733746552866.c8f8bc12489e5c0370ba59c751783a23. after waiting 0 ms 2024-12-09T12:15:54,588 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1733746552866.c8f8bc12489e5c0370ba59c751783a23. 2024-12-09T12:15:54,588 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,zzz,1733746552866.c8f8bc12489e5c0370ba59c751783a23. 2024-12-09T12:15:54,588 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for c8f8bc12489e5c0370ba59c751783a23: Waiting for close lock at 1733746554588Disabling compacts and flushes for region at 1733746554588Disabling writes for close at 1733746554588Writing region close event to WAL at 1733746554588Closed at 1733746554588 2024-12-09T12:15:54,657 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,sss,1733746552866.603cf0cbb0e068a5f7727e4abab3a5d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:54,657 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 603cf0cbb0e068a5f7727e4abab3a5d5, disabling compactions & flushes 2024-12-09T12:15:54,657 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,sss,1733746552866.603cf0cbb0e068a5f7727e4abab3a5d5. 2024-12-09T12:15:54,657 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1733746552866.603cf0cbb0e068a5f7727e4abab3a5d5. 2024-12-09T12:15:54,657 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1733746552866.603cf0cbb0e068a5f7727e4abab3a5d5. after waiting 0 ms 2024-12-09T12:15:54,657 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1733746552866.603cf0cbb0e068a5f7727e4abab3a5d5. 2024-12-09T12:15:54,657 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,sss,1733746552866.603cf0cbb0e068a5f7727e4abab3a5d5. 2024-12-09T12:15:54,657 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 603cf0cbb0e068a5f7727e4abab3a5d5: Waiting for close lock at 1733746554657Disabling compacts and flushes for region at 1733746554657Disabling writes for close at 1733746554657Writing region close event to WAL at 1733746554657Closed at 1733746554657 2024-12-09T12:15:54,666 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T12:15:54,674 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1733746552866.8bc29ffded7f1b74a1ee2986c5e611f8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733746554666"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733746554666"}]},"ts":"1733746554666"} 2024-12-09T12:15:54,674 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1733746552866.9ac6650f2a35b2c131e93ead260da106.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733746554666"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733746554666"}]},"ts":"1733746554666"} 2024-12-09T12:15:54,674 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1733746552866.cc7aaad28378d11daf90725c0f9d39f5.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733746554666"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733746554666"}]},"ts":"1733746554666"} 2024-12-09T12:15:54,674 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733746554666"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733746554666"}]},"ts":"1733746554666"} 2024-12-09T12:15:54,675 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1733746552866.1ce28e8d0ce113d2637ab328f5ffd3be.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733746554666"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733746554666"}]},"ts":"1733746554666"} 2024-12-09T12:15:54,675 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1733746552866.6b5306e854c5aec62eff62a18cec5859.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733746554666"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733746554666"}]},"ts":"1733746554666"} 2024-12-09T12:15:54,675 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1733746552866.a258d2ecd9a494e8d56406b774da28ef.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733746554666"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733746554666"}]},"ts":"1733746554666"} 2024-12-09T12:15:54,675 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1733746552866.c910d27fbf3265165c7fd59609e57c82.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733746554666"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733746554666"}]},"ts":"1733746554666"} 2024-12-09T12:15:54,675 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1733746552866.a0cf25c9917186a9e3dbd87e44f33615.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733746554666"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733746554666"}]},"ts":"1733746554666"} 2024-12-09T12:15:54,676 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1733746552866.ebee1500f62faadfac8720740a1e527e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733746554666"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733746554666"}]},"ts":"1733746554666"} 2024-12-09T12:15:54,676 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1733746552866.c9702c569de70ef36018a26943c3226f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733746554666"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733746554666"}]},"ts":"1733746554666"} 2024-12-09T12:15:54,676 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1733746552866.d9e23742363329565223433d05d86296.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733746554666"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733746554666"}]},"ts":"1733746554666"} 2024-12-09T12:15:54,677 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1733746552866.62a60a0dff0c026a110e755e6c249001.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733746554666"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733746554666"}]},"ts":"1733746554666"} 2024-12-09T12:15:54,677 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1733746552866.fdcb3d892a0c9cf641eaedbbd9ec5937.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733746554666"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733746554666"}]},"ts":"1733746554666"} 2024-12-09T12:15:54,677 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1733746552866.3a1205a417c23329b514f783f4727022.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733746554666"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733746554666"}]},"ts":"1733746554666"} 2024-12-09T12:15:54,677 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1733746552866.43f38f304815b66d00adc6292e217f1a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733746554666"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733746554666"}]},"ts":"1733746554666"} 2024-12-09T12:15:54,677 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1733746552866.fab7cda743d5cc81945ffb7c431d6346.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733746554666"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733746554666"}]},"ts":"1733746554666"} 2024-12-09T12:15:54,677 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1733746552866.e0adbf1ba01f94cad79f2814d407182f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733746554666"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733746554666"}]},"ts":"1733746554666"} 2024-12-09T12:15:54,677 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1733746552866.ba4a26e129719340e5696ac56c6b61ac.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733746554666"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733746554666"}]},"ts":"1733746554666"} 2024-12-09T12:15:54,678 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1733746552866.b0351c7e96a16dae18ff5a8d1664b2b1.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733746554666"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733746554666"}]},"ts":"1733746554666"} 2024-12-09T12:15:54,678 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1733746552866.23280f1743db631eaa35bf3487e69147.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733746554666"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733746554666"}]},"ts":"1733746554666"} 2024-12-09T12:15:54,678 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1733746552866.8ceebab6d8b44ccd74ad65e871405953.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733746554666"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733746554666"}]},"ts":"1733746554666"} 2024-12-09T12:15:54,678 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1733746552866.6a763ac99f0e3a37f94a31b5f0479530.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733746554666"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733746554666"}]},"ts":"1733746554666"} 2024-12-09T12:15:54,678 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1733746552866.7eaca6c428d8bff0cd952b3c2dfe5408.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733746554666"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733746554666"}]},"ts":"1733746554666"} 2024-12-09T12:15:54,678 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1733746552866.c8f8bc12489e5c0370ba59c751783a23.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733746554666"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733746554666"}]},"ts":"1733746554666"} 2024-12-09T12:15:54,678 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1733746552866.603cf0cbb0e068a5f7727e4abab3a5d5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733746554666"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733746554666"}]},"ts":"1733746554666"} 2024-12-09T12:15:54,778 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-12-09T12:15:54,784 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T12:15:54,788 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733746554784"}]},"ts":"1733746554784"} 2024-12-09T12:15:54,805 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-12-09T12:15:54,806 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {f15dd41c88ad=0} racks are {/default-rack=0} 2024-12-09T12:15:54,855 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T12:15:54,855 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T12:15:54,855 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-09T12:15:54,856 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T12:15:54,856 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T12:15:54,856 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-09T12:15:54,856 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T12:15:54,856 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T12:15:54,856 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-09T12:15:54,856 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T12:15:54,859 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cc7aaad28378d11daf90725c0f9d39f5, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9ac6650f2a35b2c131e93ead260da106, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1ce28e8d0ce113d2637ab328f5ffd3be, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6b5306e854c5aec62eff62a18cec5859, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c910d27fbf3265165c7fd59609e57c82, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a0cf25c9917186a9e3dbd87e44f33615, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d9e23742363329565223433d05d86296, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c9702c569de70ef36018a26943c3226f, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fdcb3d892a0c9cf641eaedbbd9ec5937, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3a1205a417c23329b514f783f4727022, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fab7cda743d5cc81945ffb7c431d6346, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e0adbf1ba01f94cad79f2814d407182f, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=603cf0cbb0e068a5f7727e4abab3a5d5, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b0351c7e96a16dae18ff5a8d1664b2b1, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=23280f1743db631eaa35bf3487e69147, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6a763ac99f0e3a37f94a31b5f0479530, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7eaca6c428d8bff0cd952b3c2dfe5408, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN}] 2024-12-09T12:15:54,871 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN 2024-12-09T12:15:54,874 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN; state=OFFLINE, location=f15dd41c88ad,33783,1733746548634; forceNewPlan=false, retain=false 2024-12-09T12:15:54,877 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7eaca6c428d8bff0cd952b3c2dfe5408, ASSIGN 2024-12-09T12:15:54,879 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7eaca6c428d8bff0cd952b3c2dfe5408, ASSIGN; state=OFFLINE, location=f15dd41c88ad,38111,1733746548467; forceNewPlan=false, retain=false 2024-12-09T12:15:54,881 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6a763ac99f0e3a37f94a31b5f0479530, ASSIGN 2024-12-09T12:15:54,884 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6a763ac99f0e3a37f94a31b5f0479530, ASSIGN; state=OFFLINE, location=f15dd41c88ad,36579,1733746548727; forceNewPlan=false, retain=false 2024-12-09T12:15:54,887 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN 2024-12-09T12:15:54,889 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN; state=OFFLINE, location=f15dd41c88ad,33783,1733746548634; forceNewPlan=false, retain=false 2024-12-09T12:15:54,891 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=23280f1743db631eaa35bf3487e69147, ASSIGN 2024-12-09T12:15:54,894 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=23280f1743db631eaa35bf3487e69147, ASSIGN; state=OFFLINE, location=f15dd41c88ad,38111,1733746548467; forceNewPlan=false, retain=false 2024-12-09T12:15:54,897 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b0351c7e96a16dae18ff5a8d1664b2b1, ASSIGN 2024-12-09T12:15:54,900 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b0351c7e96a16dae18ff5a8d1664b2b1, ASSIGN; state=OFFLINE, location=f15dd41c88ad,36579,1733746548727; forceNewPlan=false, retain=false 2024-12-09T12:15:54,903 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:15:54,915 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN; state=OFFLINE, location=f15dd41c88ad,33783,1733746548634; forceNewPlan=false, retain=false 2024-12-09T12:15:54,923 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=603cf0cbb0e068a5f7727e4abab3a5d5, ASSIGN 2024-12-09T12:15:54,929 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=603cf0cbb0e068a5f7727e4abab3a5d5, ASSIGN; state=OFFLINE, location=f15dd41c88ad,36579,1733746548727; forceNewPlan=false, retain=false 2024-12-09T12:15:54,931 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e0adbf1ba01f94cad79f2814d407182f, ASSIGN 2024-12-09T12:15:54,933 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e0adbf1ba01f94cad79f2814d407182f, ASSIGN; state=OFFLINE, location=f15dd41c88ad,38111,1733746548467; forceNewPlan=false, retain=false 2024-12-09T12:15:54,934 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fab7cda743d5cc81945ffb7c431d6346, ASSIGN 2024-12-09T12:15:54,936 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fab7cda743d5cc81945ffb7c431d6346, ASSIGN; state=OFFLINE, location=f15dd41c88ad,36579,1733746548727; forceNewPlan=false, retain=false 2024-12-09T12:15:54,938 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN 2024-12-09T12:15:54,940 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN; state=OFFLINE, location=f15dd41c88ad,33783,1733746548634; forceNewPlan=false, retain=false 2024-12-09T12:15:54,941 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3a1205a417c23329b514f783f4727022, ASSIGN 2024-12-09T12:15:54,943 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3a1205a417c23329b514f783f4727022, ASSIGN; state=OFFLINE, location=f15dd41c88ad,38111,1733746548467; forceNewPlan=false, retain=false 2024-12-09T12:15:54,945 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fdcb3d892a0c9cf641eaedbbd9ec5937, ASSIGN 2024-12-09T12:15:54,949 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fdcb3d892a0c9cf641eaedbbd9ec5937, ASSIGN; state=OFFLINE, location=f15dd41c88ad,36579,1733746548727; forceNewPlan=false, retain=false 2024-12-09T12:15:54,950 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:15:54,952 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN; state=OFFLINE, location=f15dd41c88ad,33783,1733746548634; forceNewPlan=false, retain=false 2024-12-09T12:15:54,957 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:15:54,959 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN; state=OFFLINE, location=f15dd41c88ad,33783,1733746548634; forceNewPlan=false, retain=false 2024-12-09T12:15:54,960 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c9702c569de70ef36018a26943c3226f, ASSIGN 2024-12-09T12:15:54,962 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c9702c569de70ef36018a26943c3226f, ASSIGN; state=OFFLINE, location=f15dd41c88ad,36579,1733746548727; forceNewPlan=false, retain=false 2024-12-09T12:15:54,967 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d9e23742363329565223433d05d86296, ASSIGN 2024-12-09T12:15:54,968 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d9e23742363329565223433d05d86296, ASSIGN; state=OFFLINE, location=f15dd41c88ad,38111,1733746548467; forceNewPlan=false, retain=false 2024-12-09T12:15:54,970 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a0cf25c9917186a9e3dbd87e44f33615, ASSIGN 2024-12-09T12:15:54,971 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a0cf25c9917186a9e3dbd87e44f33615, ASSIGN; state=OFFLINE, location=f15dd41c88ad,38111,1733746548467; forceNewPlan=false, retain=false 2024-12-09T12:15:54,973 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN 2024-12-09T12:15:54,974 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN; state=OFFLINE, location=f15dd41c88ad,33783,1733746548634; forceNewPlan=false, retain=false 2024-12-09T12:15:54,975 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c910d27fbf3265165c7fd59609e57c82, ASSIGN 2024-12-09T12:15:54,976 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c910d27fbf3265165c7fd59609e57c82, ASSIGN; state=OFFLINE, location=f15dd41c88ad,36579,1733746548727; forceNewPlan=false, retain=false 2024-12-09T12:15:54,977 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6b5306e854c5aec62eff62a18cec5859, ASSIGN 2024-12-09T12:15:54,979 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6b5306e854c5aec62eff62a18cec5859, ASSIGN; state=OFFLINE, location=f15dd41c88ad,38111,1733746548467; forceNewPlan=false, retain=false 2024-12-09T12:15:54,980 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, ASSIGN 2024-12-09T12:15:54,981 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, ASSIGN; state=OFFLINE, location=f15dd41c88ad,33783,1733746548634; forceNewPlan=false, retain=false 2024-12-09T12:15:54,982 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1ce28e8d0ce113d2637ab328f5ffd3be, ASSIGN 2024-12-09T12:15:54,983 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1ce28e8d0ce113d2637ab328f5ffd3be, ASSIGN; state=OFFLINE, location=f15dd41c88ad,36579,1733746548727; forceNewPlan=false, retain=false 2024-12-09T12:15:54,985 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN 2024-12-09T12:15:54,986 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN; state=OFFLINE, location=f15dd41c88ad,33783,1733746548634; forceNewPlan=false, retain=false 2024-12-09T12:15:54,988 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9ac6650f2a35b2c131e93ead260da106, ASSIGN 2024-12-09T12:15:54,989 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9ac6650f2a35b2c131e93ead260da106, ASSIGN; state=OFFLINE, location=f15dd41c88ad,36579,1733746548727; forceNewPlan=false, retain=false 2024-12-09T12:15:54,991 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cc7aaad28378d11daf90725c0f9d39f5, ASSIGN 2024-12-09T12:15:54,992 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=cc7aaad28378d11daf90725c0f9d39f5, ASSIGN; state=OFFLINE, location=f15dd41c88ad,38111,1733746548467; forceNewPlan=false, retain=false 2024-12-09T12:15:55,028 INFO [f15dd41c88ad:42519 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-12-09T12:15:55,029 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=7eaca6c428d8bff0cd952b3c2dfe5408, regionState=OPENING, regionLocation=f15dd41c88ad,38111,1733746548467 2024-12-09T12:15:55,030 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=23280f1743db631eaa35bf3487e69147, regionState=OPENING, regionLocation=f15dd41c88ad,38111,1733746548467 2024-12-09T12:15:55,031 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=e0adbf1ba01f94cad79f2814d407182f, regionState=OPENING, regionLocation=f15dd41c88ad,38111,1733746548467 2024-12-09T12:15:55,032 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=3a1205a417c23329b514f783f4727022, regionState=OPENING, regionLocation=f15dd41c88ad,38111,1733746548467 2024-12-09T12:15:55,032 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=d9e23742363329565223433d05d86296, regionState=OPENING, regionLocation=f15dd41c88ad,38111,1733746548467 2024-12-09T12:15:55,033 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=a0cf25c9917186a9e3dbd87e44f33615, regionState=OPENING, regionLocation=f15dd41c88ad,38111,1733746548467 2024-12-09T12:15:55,034 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=6b5306e854c5aec62eff62a18cec5859, regionState=OPENING, regionLocation=f15dd41c88ad,38111,1733746548467 2024-12-09T12:15:55,035 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cc7aaad28378d11daf90725c0f9d39f5, regionState=OPENING, regionLocation=f15dd41c88ad,38111,1733746548467 2024-12-09T12:15:55,037 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=6a763ac99f0e3a37f94a31b5f0479530, regionState=OPENING, regionLocation=f15dd41c88ad,36579,1733746548727 2024-12-09T12:15:55,038 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=b0351c7e96a16dae18ff5a8d1664b2b1, regionState=OPENING, regionLocation=f15dd41c88ad,36579,1733746548727 2024-12-09T12:15:55,039 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=603cf0cbb0e068a5f7727e4abab3a5d5, regionState=OPENING, regionLocation=f15dd41c88ad,36579,1733746548727 2024-12-09T12:15:55,040 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=fab7cda743d5cc81945ffb7c431d6346, regionState=OPENING, regionLocation=f15dd41c88ad,36579,1733746548727 2024-12-09T12:15:55,040 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=fdcb3d892a0c9cf641eaedbbd9ec5937, regionState=OPENING, regionLocation=f15dd41c88ad,36579,1733746548727 2024-12-09T12:15:55,041 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=c9702c569de70ef36018a26943c3226f, regionState=OPENING, regionLocation=f15dd41c88ad,36579,1733746548727 2024-12-09T12:15:55,042 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7eaca6c428d8bff0cd952b3c2dfe5408, ASSIGN because future has completed 2024-12-09T12:15:55,042 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=c910d27fbf3265165c7fd59609e57c82, regionState=OPENING, regionLocation=f15dd41c88ad,36579,1733746548727 2024-12-09T12:15:55,045 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1ce28e8d0ce113d2637ab328f5ffd3be, regionState=OPENING, regionLocation=f15dd41c88ad,36579,1733746548727 2024-12-09T12:15:55,045 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=9ac6650f2a35b2c131e93ead260da106, regionState=OPENING, regionLocation=f15dd41c88ad,36579,1733746548727 2024-12-09T12:15:55,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7eaca6c428d8bff0cd952b3c2dfe5408, server=f15dd41c88ad,38111,1733746548467}] 2024-12-09T12:15:55,052 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=23280f1743db631eaa35bf3487e69147, ASSIGN because future has completed 2024-12-09T12:15:55,063 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e0adbf1ba01f94cad79f2814d407182f, ASSIGN because future has completed 2024-12-09T12:15:55,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3a1205a417c23329b514f783f4727022, ASSIGN because future has completed 2024-12-09T12:15:55,081 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d9e23742363329565223433d05d86296, ASSIGN because future has completed 2024-12-09T12:15:55,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a1205a417c23329b514f783f4727022, server=f15dd41c88ad,38111,1733746548467}] 2024-12-09T12:15:55,088 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a0cf25c9917186a9e3dbd87e44f33615, ASSIGN because future has completed 2024-12-09T12:15:55,095 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6b5306e854c5aec62eff62a18cec5859, ASSIGN because future has completed 2024-12-09T12:15:55,096 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=cc7aaad28378d11daf90725c0f9d39f5, ASSIGN because future has completed 2024-12-09T12:15:55,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cc7aaad28378d11daf90725c0f9d39f5, server=f15dd41c88ad,38111,1733746548467}] 2024-12-09T12:15:55,111 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6a763ac99f0e3a37f94a31b5f0479530, ASSIGN because future has completed 2024-12-09T12:15:55,113 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b0351c7e96a16dae18ff5a8d1664b2b1, ASSIGN because future has completed 2024-12-09T12:15:55,114 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=603cf0cbb0e068a5f7727e4abab3a5d5, ASSIGN because future has completed 2024-12-09T12:15:55,116 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fab7cda743d5cc81945ffb7c431d6346, ASSIGN because future has completed 2024-12-09T12:15:55,119 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fdcb3d892a0c9cf641eaedbbd9ec5937, ASSIGN because future has completed 2024-12-09T12:15:55,122 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c9702c569de70ef36018a26943c3226f, ASSIGN because future has completed 2024-12-09T12:15:55,124 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c910d27fbf3265165c7fd59609e57c82, ASSIGN because future has completed 2024-12-09T12:15:55,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1ce28e8d0ce113d2637ab328f5ffd3be, ASSIGN because future has completed 2024-12-09T12:15:55,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ce28e8d0ce113d2637ab328f5ffd3be, server=f15dd41c88ad,36579,1733746548727}] 2024-12-09T12:15:55,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9ac6650f2a35b2c131e93ead260da106, ASSIGN because future has completed 2024-12-09T12:15:55,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T12:15:55,138 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ac6650f2a35b2c131e93ead260da106, server=f15dd41c88ad,36579,1733746548727}] 2024-12-09T12:15:55,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure c910d27fbf3265165c7fd59609e57c82, server=f15dd41c88ad,36579,1733746548727}] 2024-12-09T12:15:55,151 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure c9702c569de70ef36018a26943c3226f, server=f15dd41c88ad,36579,1733746548727}] 2024-12-09T12:15:55,159 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure fdcb3d892a0c9cf641eaedbbd9ec5937, server=f15dd41c88ad,36579,1733746548727}] 2024-12-09T12:15:55,169 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure fab7cda743d5cc81945ffb7c431d6346, server=f15dd41c88ad,36579,1733746548727}] 2024-12-09T12:15:55,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 603cf0cbb0e068a5f7727e4abab3a5d5, server=f15dd41c88ad,36579,1733746548727}] 2024-12-09T12:15:55,190 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure b0351c7e96a16dae18ff5a8d1664b2b1, server=f15dd41c88ad,36579,1733746548727}] 2024-12-09T12:15:55,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a763ac99f0e3a37f94a31b5f0479530, server=f15dd41c88ad,36579,1733746548727}] 2024-12-09T12:15:55,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6b5306e854c5aec62eff62a18cec5859, server=f15dd41c88ad,38111,1733746548467}] 2024-12-09T12:15:55,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0cf25c9917186a9e3dbd87e44f33615, server=f15dd41c88ad,38111,1733746548467}] 2024-12-09T12:15:55,217 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T12:15:55,227 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure d9e23742363329565223433d05d86296, server=f15dd41c88ad,38111,1733746548467}] 2024-12-09T12:15:55,229 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59479, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T12:15:55,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0adbf1ba01f94cad79f2814d407182f, server=f15dd41c88ad,38111,1733746548467}] 2024-12-09T12:15:55,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23280f1743db631eaa35bf3487e69147, server=f15dd41c88ad,38111,1733746548467}] 2024-12-09T12:15:55,259 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,ooo,1733746552866.3a1205a417c23329b514f783f4727022. 2024-12-09T12:15:55,259 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 3a1205a417c23329b514f783f4727022, NAME => 'test,ooo,1733746552866.3a1205a417c23329b514f783f4727022.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-12-09T12:15:55,260 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3a1205a417c23329b514f783f4727022 2024-12-09T12:15:55,260 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,ooo,1733746552866.3a1205a417c23329b514f783f4727022.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:55,260 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 3a1205a417c23329b514f783f4727022 2024-12-09T12:15:55,260 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 3a1205a417c23329b514f783f4727022 2024-12-09T12:15:55,282 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=c8f8bc12489e5c0370ba59c751783a23, regionState=OPENING, regionLocation=f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:55,284 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=8ceebab6d8b44ccd74ad65e871405953, regionState=OPENING, regionLocation=f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:55,284 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=ba4a26e129719340e5696ac56c6b61ac, regionState=OPENING, regionLocation=f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:55,285 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=43f38f304815b66d00adc6292e217f1a, regionState=OPENING, regionLocation=f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:55,286 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=62a60a0dff0c026a110e755e6c249001, regionState=OPENING, regionLocation=f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:55,286 INFO [StoreOpener-3a1205a417c23329b514f783f4727022-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3a1205a417c23329b514f783f4727022 2024-12-09T12:15:55,287 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=ebee1500f62faadfac8720740a1e527e, regionState=OPENING, regionLocation=f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:55,288 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=a258d2ecd9a494e8d56406b774da28ef, regionState=OPENING, regionLocation=f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:55,289 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN because future has completed 2024-12-09T12:15:55,291 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T12:15:55,294 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50183, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T12:15:55,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN because future has completed 2024-12-09T12:15:55,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ceebab6d8b44ccd74ad65e871405953, server=f15dd41c88ad,33783,1733746548634}] 2024-12-09T12:15:55,307 INFO [StoreOpener-3a1205a417c23329b514f783f4727022-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3a1205a417c23329b514f783f4727022 columnFamilyName family 2024-12-09T12:15:55,307 DEBUG [StoreOpener-3a1205a417c23329b514f783f4727022-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:55,313 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN because future has completed 2024-12-09T12:15:55,314 INFO [StoreOpener-3a1205a417c23329b514f783f4727022-1 {}] regionserver.HStore(327): Store=3a1205a417c23329b514f783f4727022/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:55,317 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 3a1205a417c23329b514f783f4727022 2024-12-09T12:15:55,319 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN because future has completed 2024-12-09T12:15:55,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN because future has completed 2024-12-09T12:15:55,322 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/3a1205a417c23329b514f783f4727022 2024-12-09T12:15:55,323 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,ddd,1733746552866.1ce28e8d0ce113d2637ab328f5ffd3be. 2024-12-09T12:15:55,323 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/3a1205a417c23329b514f783f4727022 2024-12-09T12:15:55,323 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 1ce28e8d0ce113d2637ab328f5ffd3be, NAME => 'test,ddd,1733746552866.1ce28e8d0ce113d2637ab328f5ffd3be.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-12-09T12:15:55,324 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1ce28e8d0ce113d2637ab328f5ffd3be 2024-12-09T12:15:55,324 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,ddd,1733746552866.1ce28e8d0ce113d2637ab328f5ffd3be.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:55,324 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 1ce28e8d0ce113d2637ab328f5ffd3be 2024-12-09T12:15:55,324 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 3a1205a417c23329b514f783f4727022 2024-12-09T12:15:55,324 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 1ce28e8d0ce113d2637ab328f5ffd3be 2024-12-09T12:15:55,324 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 3a1205a417c23329b514f783f4727022 2024-12-09T12:15:55,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,33783,1733746548634}] 2024-12-09T12:15:55,327 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN because future has completed 2024-12-09T12:15:55,330 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 3a1205a417c23329b514f783f4727022 2024-12-09T12:15:55,331 INFO [StoreOpener-1ce28e8d0ce113d2637ab328f5ffd3be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1ce28e8d0ce113d2637ab328f5ffd3be 2024-12-09T12:15:55,331 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN because future has completed 2024-12-09T12:15:55,334 INFO [StoreOpener-1ce28e8d0ce113d2637ab328f5ffd3be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ce28e8d0ce113d2637ab328f5ffd3be columnFamilyName family 2024-12-09T12:15:55,334 DEBUG [StoreOpener-1ce28e8d0ce113d2637ab328f5ffd3be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:55,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure a258d2ecd9a494e8d56406b774da28ef, server=f15dd41c88ad,33783,1733746548634}] 2024-12-09T12:15:55,341 INFO [StoreOpener-1ce28e8d0ce113d2637ab328f5ffd3be-1 {}] regionserver.HStore(327): Store=1ce28e8d0ce113d2637ab328f5ffd3be/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:55,344 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 1ce28e8d0ce113d2637ab328f5ffd3be 2024-12-09T12:15:55,346 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/1ce28e8d0ce113d2637ab328f5ffd3be 2024-12-09T12:15:55,347 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/1ce28e8d0ce113d2637ab328f5ffd3be 2024-12-09T12:15:55,348 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 1ce28e8d0ce113d2637ab328f5ffd3be 2024-12-09T12:15:55,348 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 1ce28e8d0ce113d2637ab328f5ffd3be 2024-12-09T12:15:55,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure ebee1500f62faadfac8720740a1e527e, server=f15dd41c88ad,33783,1733746548634}] 2024-12-09T12:15:55,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43f38f304815b66d00adc6292e217f1a, server=f15dd41c88ad,33783,1733746548634}] 2024-12-09T12:15:55,358 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 1ce28e8d0ce113d2637ab328f5ffd3be 2024-12-09T12:15:55,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba4a26e129719340e5696ac56c6b61ac, server=f15dd41c88ad,33783,1733746548634}] 2024-12-09T12:15:55,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8f8bc12489e5c0370ba59c751783a23, server=f15dd41c88ad,33783,1733746548634}] 2024-12-09T12:15:55,376 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/3a1205a417c23329b514f783f4727022/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T12:15:55,377 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 3a1205a417c23329b514f783f4727022; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59367471, jitterRate=-0.11535574495792389}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:15:55,378 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3a1205a417c23329b514f783f4727022 2024-12-09T12:15:55,384 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 3a1205a417c23329b514f783f4727022: Running coprocessor pre-open hook at 1733746555264Writing region info on filesystem at 1733746555264Initializing all the Stores at 1733746555283 (+19 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746555283Cleaning up temporary data from old regions at 1733746555324 (+41 ms)Running coprocessor post-open hooks at 1733746555378 (+54 ms)Region opened successfully at 1733746555383 (+5 ms) 2024-12-09T12:15:55,386 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1733746552866.3a1205a417c23329b514f783f4727022., pid=32, masterSystemTime=1733746555217 2024-12-09T12:15:55,390 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/1ce28e8d0ce113d2637ab328f5ffd3be/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T12:15:55,391 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 1ce28e8d0ce113d2637ab328f5ffd3be; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73857795, jitterRate=0.10056690871715546}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:15:55,391 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1ce28e8d0ce113d2637ab328f5ffd3be 2024-12-09T12:15:55,391 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 1ce28e8d0ce113d2637ab328f5ffd3be: Running coprocessor pre-open hook at 1733746555324Writing region info on filesystem at 1733746555324Initializing all the Stores at 1733746555328 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746555328Cleaning up temporary data from old regions at 1733746555348 (+20 ms)Running coprocessor post-open hooks at 1733746555391 (+43 ms)Region opened successfully at 1733746555391 2024-12-09T12:15:55,393 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1733746552866.1ce28e8d0ce113d2637ab328f5ffd3be., pid=34, masterSystemTime=1733746555291 2024-12-09T12:15:55,399 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=e5251eee3971dde9f0f276cfd8632c21, regionState=OPENING, regionLocation=f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:55,401 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1733746552866.3a1205a417c23329b514f783f4727022. 2024-12-09T12:15:55,401 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,ooo,1733746552866.3a1205a417c23329b514f783f4727022. 2024-12-09T12:15:55,401 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,,1733746552866.cc7aaad28378d11daf90725c0f9d39f5. 2024-12-09T12:15:55,401 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => cc7aaad28378d11daf90725c0f9d39f5, NAME => 'test,,1733746552866.cc7aaad28378d11daf90725c0f9d39f5.', STARTKEY => '', ENDKEY => 'bbb'} 2024-12-09T12:15:55,402 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test cc7aaad28378d11daf90725c0f9d39f5 2024-12-09T12:15:55,402 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,,1733746552866.cc7aaad28378d11daf90725c0f9d39f5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:55,403 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for cc7aaad28378d11daf90725c0f9d39f5 2024-12-09T12:15:55,403 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for cc7aaad28378d11daf90725c0f9d39f5 2024-12-09T12:15:55,405 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1733746552866.1ce28e8d0ce113d2637ab328f5ffd3be. 2024-12-09T12:15:55,405 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,ddd,1733746552866.1ce28e8d0ce113d2637ab328f5ffd3be. 2024-12-09T12:15:55,405 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,qqq,1733746552866.fab7cda743d5cc81945ffb7c431d6346. 2024-12-09T12:15:55,405 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => fab7cda743d5cc81945ffb7c431d6346, NAME => 'test,qqq,1733746552866.fab7cda743d5cc81945ffb7c431d6346.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-12-09T12:15:55,406 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fab7cda743d5cc81945ffb7c431d6346 2024-12-09T12:15:55,406 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,qqq,1733746552866.fab7cda743d5cc81945ffb7c431d6346.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:55,406 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for fab7cda743d5cc81945ffb7c431d6346 2024-12-09T12:15:55,406 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for fab7cda743d5cc81945ffb7c431d6346 2024-12-09T12:15:55,407 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, ASSIGN because future has completed 2024-12-09T12:15:55,408 INFO [StoreOpener-cc7aaad28378d11daf90725c0f9d39f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region cc7aaad28378d11daf90725c0f9d39f5 2024-12-09T12:15:55,409 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=3a1205a417c23329b514f783f4727022, regionState=OPEN, openSeqNum=2, regionLocation=f15dd41c88ad,38111,1733746548467 2024-12-09T12:15:55,411 INFO [StoreOpener-cc7aaad28378d11daf90725c0f9d39f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc7aaad28378d11daf90725c0f9d39f5 columnFamilyName family 2024-12-09T12:15:55,414 DEBUG [StoreOpener-cc7aaad28378d11daf90725c0f9d39f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:55,415 INFO [StoreOpener-cc7aaad28378d11daf90725c0f9d39f5-1 {}] regionserver.HStore(327): Store=cc7aaad28378d11daf90725c0f9d39f5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:55,416 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for cc7aaad28378d11daf90725c0f9d39f5 2024-12-09T12:15:55,417 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/cc7aaad28378d11daf90725c0f9d39f5 2024-12-09T12:15:55,418 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/cc7aaad28378d11daf90725c0f9d39f5 2024-12-09T12:15:55,418 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for cc7aaad28378d11daf90725c0f9d39f5 2024-12-09T12:15:55,418 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for cc7aaad28378d11daf90725c0f9d39f5 2024-12-09T12:15:55,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a1205a417c23329b514f783f4727022, server=f15dd41c88ad,38111,1733746548467 because future has completed 2024-12-09T12:15:55,416 INFO [StoreOpener-fab7cda743d5cc81945ffb7c431d6346-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fab7cda743d5cc81945ffb7c431d6346 2024-12-09T12:15:55,422 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for cc7aaad28378d11daf90725c0f9d39f5 2024-12-09T12:15:55,427 INFO [StoreOpener-fab7cda743d5cc81945ffb7c431d6346-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fab7cda743d5cc81945ffb7c431d6346 columnFamilyName family 2024-12-09T12:15:55,428 DEBUG [StoreOpener-fab7cda743d5cc81945ffb7c431d6346-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:55,429 INFO [StoreOpener-fab7cda743d5cc81945ffb7c431d6346-1 {}] regionserver.HStore(327): Store=fab7cda743d5cc81945ffb7c431d6346/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:55,429 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for fab7cda743d5cc81945ffb7c431d6346 2024-12-09T12:15:55,430 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/fab7cda743d5cc81945ffb7c431d6346 2024-12-09T12:15:55,431 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/fab7cda743d5cc81945ffb7c431d6346 2024-12-09T12:15:55,432 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for fab7cda743d5cc81945ffb7c431d6346 2024-12-09T12:15:55,432 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for fab7cda743d5cc81945ffb7c431d6346 2024-12-09T12:15:55,435 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for fab7cda743d5cc81945ffb7c431d6346 2024-12-09T12:15:55,443 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/cc7aaad28378d11daf90725c0f9d39f5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T12:15:55,445 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened cc7aaad28378d11daf90725c0f9d39f5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65289594, jitterRate=-0.027109235525131226}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:15:55,445 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cc7aaad28378d11daf90725c0f9d39f5 2024-12-09T12:15:55,445 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for cc7aaad28378d11daf90725c0f9d39f5: Running coprocessor pre-open hook at 1733746555403Writing region info on filesystem at 1733746555403Initializing all the Stores at 1733746555408 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746555408Cleaning up temporary data from old regions at 1733746555419 (+11 ms)Running coprocessor post-open hooks at 1733746555445 (+26 ms)Region opened successfully at 1733746555445 2024-12-09T12:15:55,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=19 2024-12-09T12:15:55,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 3a1205a417c23329b514f783f4727022, server=f15dd41c88ad,38111,1733746548467 in 351 msec 2024-12-09T12:15:55,447 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1733746552866.cc7aaad28378d11daf90725c0f9d39f5., pid=33, masterSystemTime=1733746555217 2024-12-09T12:15:55,447 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/fab7cda743d5cc81945ffb7c431d6346/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T12:15:55,448 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened fab7cda743d5cc81945ffb7c431d6346; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60932380, jitterRate=-0.09203678369522095}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:15:55,448 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fab7cda743d5cc81945ffb7c431d6346 2024-12-09T12:15:55,449 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for fab7cda743d5cc81945ffb7c431d6346: Running coprocessor pre-open hook at 1733746555406Writing region info on filesystem at 1733746555406Initializing all the Stores at 1733746555414 (+8 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746555414Cleaning up temporary data from old regions at 1733746555432 (+18 ms)Running coprocessor post-open hooks at 1733746555448 (+16 ms)Region opened successfully at 1733746555448 2024-12-09T12:15:55,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3a1205a417c23329b514f783f4727022, ASSIGN in 587 msec 2024-12-09T12:15:55,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure e5251eee3971dde9f0f276cfd8632c21, server=f15dd41c88ad,33783,1733746548634}] 2024-12-09T12:15:55,450 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1733746552866.fab7cda743d5cc81945ffb7c431d6346., pid=39, masterSystemTime=1733746555291 2024-12-09T12:15:55,451 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1733746552866.cc7aaad28378d11daf90725c0f9d39f5. 2024-12-09T12:15:55,451 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,,1733746552866.cc7aaad28378d11daf90725c0f9d39f5. 2024-12-09T12:15:55,451 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,yyy,1733746552866.7eaca6c428d8bff0cd952b3c2dfe5408. 2024-12-09T12:15:55,451 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 7eaca6c428d8bff0cd952b3c2dfe5408, NAME => 'test,yyy,1733746552866.7eaca6c428d8bff0cd952b3c2dfe5408.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-12-09T12:15:55,452 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7eaca6c428d8bff0cd952b3c2dfe5408 2024-12-09T12:15:55,452 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,yyy,1733746552866.7eaca6c428d8bff0cd952b3c2dfe5408.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:55,452 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 7eaca6c428d8bff0cd952b3c2dfe5408 2024-12-09T12:15:55,452 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 7eaca6c428d8bff0cd952b3c2dfe5408 2024-12-09T12:15:55,453 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1733746552866.fab7cda743d5cc81945ffb7c431d6346. 2024-12-09T12:15:55,453 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,qqq,1733746552866.fab7cda743d5cc81945ffb7c431d6346. 2024-12-09T12:15:55,453 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,kkk,1733746552866.c9702c569de70ef36018a26943c3226f. 2024-12-09T12:15:55,453 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => c9702c569de70ef36018a26943c3226f, NAME => 'test,kkk,1733746552866.c9702c569de70ef36018a26943c3226f.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-12-09T12:15:55,454 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c9702c569de70ef36018a26943c3226f 2024-12-09T12:15:55,454 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,kkk,1733746552866.c9702c569de70ef36018a26943c3226f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:55,454 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for c9702c569de70ef36018a26943c3226f 2024-12-09T12:15:55,454 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for c9702c569de70ef36018a26943c3226f 2024-12-09T12:15:55,456 INFO [StoreOpener-c9702c569de70ef36018a26943c3226f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c9702c569de70ef36018a26943c3226f 2024-12-09T12:15:55,457 INFO [StoreOpener-7eaca6c428d8bff0cd952b3c2dfe5408-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7eaca6c428d8bff0cd952b3c2dfe5408 2024-12-09T12:15:55,459 INFO [StoreOpener-c9702c569de70ef36018a26943c3226f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c9702c569de70ef36018a26943c3226f columnFamilyName family 2024-12-09T12:15:55,459 DEBUG [StoreOpener-c9702c569de70ef36018a26943c3226f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:55,459 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=fab7cda743d5cc81945ffb7c431d6346, regionState=OPEN, openSeqNum=2, regionLocation=f15dd41c88ad,36579,1733746548727 2024-12-09T12:15:55,460 INFO [StoreOpener-c9702c569de70ef36018a26943c3226f-1 {}] regionserver.HStore(327): Store=c9702c569de70ef36018a26943c3226f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:55,460 INFO [StoreOpener-7eaca6c428d8bff0cd952b3c2dfe5408-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7eaca6c428d8bff0cd952b3c2dfe5408 columnFamilyName family 2024-12-09T12:15:55,460 DEBUG [StoreOpener-7eaca6c428d8bff0cd952b3c2dfe5408-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:55,460 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for c9702c569de70ef36018a26943c3226f 2024-12-09T12:15:55,460 INFO [StoreOpener-7eaca6c428d8bff0cd952b3c2dfe5408-1 {}] regionserver.HStore(327): Store=7eaca6c428d8bff0cd952b3c2dfe5408/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:55,461 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 7eaca6c428d8bff0cd952b3c2dfe5408 2024-12-09T12:15:55,463 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/c9702c569de70ef36018a26943c3226f 2024-12-09T12:15:55,463 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/7eaca6c428d8bff0cd952b3c2dfe5408 2024-12-09T12:15:55,463 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/c9702c569de70ef36018a26943c3226f 2024-12-09T12:15:55,464 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for c9702c569de70ef36018a26943c3226f 2024-12-09T12:15:55,464 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/7eaca6c428d8bff0cd952b3c2dfe5408 2024-12-09T12:15:55,464 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for c9702c569de70ef36018a26943c3226f 2024-12-09T12:15:55,465 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cc7aaad28378d11daf90725c0f9d39f5, regionState=OPEN, openSeqNum=2, regionLocation=f15dd41c88ad,38111,1733746548467 2024-12-09T12:15:55,465 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 7eaca6c428d8bff0cd952b3c2dfe5408 2024-12-09T12:15:55,465 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 7eaca6c428d8bff0cd952b3c2dfe5408 2024-12-09T12:15:55,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure fab7cda743d5cc81945ffb7c431d6346, server=f15dd41c88ad,36579,1733746548727 because future has completed 2024-12-09T12:15:55,469 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cc7aaad28378d11daf90725c0f9d39f5, server=f15dd41c88ad,38111,1733746548467 because future has completed 2024-12-09T12:15:55,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=21 2024-12-09T12:15:55,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure fab7cda743d5cc81945ffb7c431d6346, server=f15dd41c88ad,36579,1733746548727 in 300 msec 2024-12-09T12:15:55,475 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for c9702c569de70ef36018a26943c3226f 2024-12-09T12:15:55,476 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 7eaca6c428d8bff0cd952b3c2dfe5408 2024-12-09T12:15:55,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fab7cda743d5cc81945ffb7c431d6346, ASSIGN in 616 msec 2024-12-09T12:15:55,479 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,www,1733746552866.8ceebab6d8b44ccd74ad65e871405953. 2024-12-09T12:15:55,480 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 8ceebab6d8b44ccd74ad65e871405953, NAME => 'test,www,1733746552866.8ceebab6d8b44ccd74ad65e871405953.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-12-09T12:15:55,480 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8ceebab6d8b44ccd74ad65e871405953 2024-12-09T12:15:55,481 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,www,1733746552866.8ceebab6d8b44ccd74ad65e871405953.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:55,481 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 8ceebab6d8b44ccd74ad65e871405953 2024-12-09T12:15:55,481 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 8ceebab6d8b44ccd74ad65e871405953 2024-12-09T12:15:55,482 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/c9702c569de70ef36018a26943c3226f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T12:15:55,483 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened c9702c569de70ef36018a26943c3226f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68998909, jitterRate=0.028163865208625793}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:15:55,484 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c9702c569de70ef36018a26943c3226f 2024-12-09T12:15:55,484 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for c9702c569de70ef36018a26943c3226f: Running coprocessor pre-open hook at 1733746555454Writing region info on filesystem at 1733746555454Initializing all the Stores at 1733746555456 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746555456Cleaning up temporary data from old regions at 1733746555464 (+8 ms)Running coprocessor post-open hooks at 1733746555484 (+20 ms)Region opened successfully at 1733746555484 2024-12-09T12:15:55,485 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/7eaca6c428d8bff0cd952b3c2dfe5408/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T12:15:55,486 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 7eaca6c428d8bff0cd952b3c2dfe5408; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72865896, jitterRate=0.08578646183013916}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:15:55,486 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7eaca6c428d8bff0cd952b3c2dfe5408 2024-12-09T12:15:55,486 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 7eaca6c428d8bff0cd952b3c2dfe5408: Running coprocessor pre-open hook at 1733746555453Writing region info on filesystem at 1733746555453Initializing all the Stores at 1733746555457 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746555457Cleaning up temporary data from old regions at 1733746555465 (+8 ms)Running coprocessor post-open hooks at 1733746555486 (+21 ms)Region opened successfully at 1733746555486 2024-12-09T12:15:55,487 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1733746552866.c9702c569de70ef36018a26943c3226f., pid=37, masterSystemTime=1733746555291 2024-12-09T12:15:55,487 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1733746552866.7eaca6c428d8bff0cd952b3c2dfe5408., pid=31, masterSystemTime=1733746555217 2024-12-09T12:15:55,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=5 2024-12-09T12:15:55,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure cc7aaad28378d11daf90725c0f9d39f5, server=f15dd41c88ad,38111,1733746548467 in 372 msec 2024-12-09T12:15:55,491 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1733746552866.c9702c569de70ef36018a26943c3226f. 2024-12-09T12:15:55,491 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,kkk,1733746552866.c9702c569de70ef36018a26943c3226f. 2024-12-09T12:15:55,491 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,xxx,1733746552866.6a763ac99f0e3a37f94a31b5f0479530. 2024-12-09T12:15:55,491 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 6a763ac99f0e3a37f94a31b5f0479530, NAME => 'test,xxx,1733746552866.6a763ac99f0e3a37f94a31b5f0479530.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-12-09T12:15:55,492 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6a763ac99f0e3a37f94a31b5f0479530 2024-12-09T12:15:55,492 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,xxx,1733746552866.6a763ac99f0e3a37f94a31b5f0479530.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:55,492 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 6a763ac99f0e3a37f94a31b5f0479530 2024-12-09T12:15:55,492 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 6a763ac99f0e3a37f94a31b5f0479530 2024-12-09T12:15:55,493 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=c9702c569de70ef36018a26943c3226f, regionState=OPEN, openSeqNum=2, regionLocation=f15dd41c88ad,36579,1733746548727 2024-12-09T12:15:55,493 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1733746552866.7eaca6c428d8bff0cd952b3c2dfe5408. 2024-12-09T12:15:55,493 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,yyy,1733746552866.7eaca6c428d8bff0cd952b3c2dfe5408. 2024-12-09T12:15:55,493 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,rrr,1733746552866.e0adbf1ba01f94cad79f2814d407182f. 2024-12-09T12:15:55,494 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => e0adbf1ba01f94cad79f2814d407182f, NAME => 'test,rrr,1733746552866.e0adbf1ba01f94cad79f2814d407182f.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-12-09T12:15:55,494 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e0adbf1ba01f94cad79f2814d407182f 2024-12-09T12:15:55,494 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,rrr,1733746552866.e0adbf1ba01f94cad79f2814d407182f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:55,494 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for e0adbf1ba01f94cad79f2814d407182f 2024-12-09T12:15:55,494 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for e0adbf1ba01f94cad79f2814d407182f 2024-12-09T12:15:55,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure c9702c569de70ef36018a26943c3226f, server=f15dd41c88ad,36579,1733746548727 because future has completed 2024-12-09T12:15:55,502 INFO [StoreOpener-6a763ac99f0e3a37f94a31b5f0479530-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6a763ac99f0e3a37f94a31b5f0479530 2024-12-09T12:15:55,503 INFO [StoreOpener-e0adbf1ba01f94cad79f2814d407182f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e0adbf1ba01f94cad79f2814d407182f 2024-12-09T12:15:55,505 INFO [StoreOpener-6a763ac99f0e3a37f94a31b5f0479530-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a763ac99f0e3a37f94a31b5f0479530 columnFamilyName family 2024-12-09T12:15:55,505 INFO [StoreOpener-8ceebab6d8b44ccd74ad65e871405953-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8ceebab6d8b44ccd74ad65e871405953 2024-12-09T12:15:55,505 DEBUG [StoreOpener-6a763ac99f0e3a37f94a31b5f0479530-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:55,507 INFO [StoreOpener-e0adbf1ba01f94cad79f2814d407182f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e0adbf1ba01f94cad79f2814d407182f columnFamilyName family 2024-12-09T12:15:55,507 DEBUG [StoreOpener-e0adbf1ba01f94cad79f2814d407182f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:55,508 INFO [StoreOpener-8ceebab6d8b44ccd74ad65e871405953-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ceebab6d8b44ccd74ad65e871405953 columnFamilyName family 2024-12-09T12:15:55,508 DEBUG [StoreOpener-8ceebab6d8b44ccd74ad65e871405953-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:55,509 INFO [StoreOpener-6a763ac99f0e3a37f94a31b5f0479530-1 {}] regionserver.HStore(327): Store=6a763ac99f0e3a37f94a31b5f0479530/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:55,509 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 6a763ac99f0e3a37f94a31b5f0479530 2024-12-09T12:15:55,509 INFO [StoreOpener-e0adbf1ba01f94cad79f2814d407182f-1 {}] regionserver.HStore(327): Store=e0adbf1ba01f94cad79f2814d407182f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:55,510 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for e0adbf1ba01f94cad79f2814d407182f 2024-12-09T12:15:55,510 INFO [StoreOpener-8ceebab6d8b44ccd74ad65e871405953-1 {}] regionserver.HStore(327): Store=8ceebab6d8b44ccd74ad65e871405953/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:55,510 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/6a763ac99f0e3a37f94a31b5f0479530 2024-12-09T12:15:55,510 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 8ceebab6d8b44ccd74ad65e871405953 2024-12-09T12:15:55,511 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/e0adbf1ba01f94cad79f2814d407182f 2024-12-09T12:15:55,511 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/6a763ac99f0e3a37f94a31b5f0479530 2024-12-09T12:15:55,512 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 6a763ac99f0e3a37f94a31b5f0479530 2024-12-09T12:15:55,512 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/e0adbf1ba01f94cad79f2814d407182f 2024-12-09T12:15:55,512 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/8ceebab6d8b44ccd74ad65e871405953 2024-12-09T12:15:55,512 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 6a763ac99f0e3a37f94a31b5f0479530 2024-12-09T12:15:55,512 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/8ceebab6d8b44ccd74ad65e871405953 2024-12-09T12:15:55,513 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for e0adbf1ba01f94cad79f2814d407182f 2024-12-09T12:15:55,513 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for e0adbf1ba01f94cad79f2814d407182f 2024-12-09T12:15:55,513 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 8ceebab6d8b44ccd74ad65e871405953 2024-12-09T12:15:55,513 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 8ceebab6d8b44ccd74ad65e871405953 2024-12-09T12:15:55,515 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 6a763ac99f0e3a37f94a31b5f0479530 2024-12-09T12:15:55,516 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 8ceebab6d8b44ccd74ad65e871405953 2024-12-09T12:15:55,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=15 2024-12-09T12:15:55,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure c9702c569de70ef36018a26943c3226f, server=f15dd41c88ad,36579,1733746548727 in 348 msec 2024-12-09T12:15:55,517 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for e0adbf1ba01f94cad79f2814d407182f 2024-12-09T12:15:55,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c9702c569de70ef36018a26943c3226f, ASSIGN in 658 msec 2024-12-09T12:15:55,522 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=7eaca6c428d8bff0cd952b3c2dfe5408, regionState=OPEN, openSeqNum=2, regionLocation=f15dd41c88ad,38111,1733746548467 2024-12-09T12:15:55,523 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/6a763ac99f0e3a37f94a31b5f0479530/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T12:15:55,524 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 6a763ac99f0e3a37f94a31b5f0479530; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68048867, jitterRate=0.014007136225700378}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:15:55,524 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6a763ac99f0e3a37f94a31b5f0479530 2024-12-09T12:15:55,525 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 6a763ac99f0e3a37f94a31b5f0479530: Running coprocessor pre-open hook at 1733746555492Writing region info on filesystem at 1733746555492Initializing all the Stores at 1733746555495 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746555495Cleaning up temporary data from old regions at 1733746555512 (+17 ms)Running coprocessor post-open hooks at 1733746555524 (+12 ms)Region opened successfully at 1733746555524 2024-12-09T12:15:55,526 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/8ceebab6d8b44ccd74ad65e871405953/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T12:15:55,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7eaca6c428d8bff0cd952b3c2dfe5408, server=f15dd41c88ad,38111,1733746548467 because future has completed 2024-12-09T12:15:55,527 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 8ceebab6d8b44ccd74ad65e871405953; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60673578, jitterRate=-0.09589323401451111}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:15:55,527 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8ceebab6d8b44ccd74ad65e871405953 2024-12-09T12:15:55,527 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 8ceebab6d8b44ccd74ad65e871405953: Running coprocessor pre-open hook at 1733746555481Writing region info on filesystem at 1733746555481Initializing all the Stores at 1733746555484 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746555484Cleaning up temporary data from old regions at 1733746555513 (+29 ms)Running coprocessor post-open hooks at 1733746555527 (+14 ms)Region opened successfully at 1733746555527 2024-12-09T12:15:55,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cc7aaad28378d11daf90725c0f9d39f5, ASSIGN in 666 msec 2024-12-09T12:15:55,529 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1733746552866.8ceebab6d8b44ccd74ad65e871405953., pid=48, masterSystemTime=1733746555465 2024-12-09T12:15:55,533 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1733746552866.8ceebab6d8b44ccd74ad65e871405953. 2024-12-09T12:15:55,533 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,www,1733746552866.8ceebab6d8b44ccd74ad65e871405953. 2024-12-09T12:15:55,533 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,zzz,1733746552866.c8f8bc12489e5c0370ba59c751783a23. 2024-12-09T12:15:55,533 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1733746552866.6a763ac99f0e3a37f94a31b5f0479530., pid=42, masterSystemTime=1733746555291 2024-12-09T12:15:55,533 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => c8f8bc12489e5c0370ba59c751783a23, NAME => 'test,zzz,1733746552866.c8f8bc12489e5c0370ba59c751783a23.', STARTKEY => 'zzz', ENDKEY => ''} 2024-12-09T12:15:55,534 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c8f8bc12489e5c0370ba59c751783a23 2024-12-09T12:15:55,534 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,zzz,1733746552866.c8f8bc12489e5c0370ba59c751783a23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:55,534 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for c8f8bc12489e5c0370ba59c751783a23 2024-12-09T12:15:55,534 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for c8f8bc12489e5c0370ba59c751783a23 2024-12-09T12:15:55,535 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/e0adbf1ba01f94cad79f2814d407182f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T12:15:55,536 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened e0adbf1ba01f94cad79f2814d407182f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64868374, jitterRate=-0.033385902643203735}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:15:55,536 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e0adbf1ba01f94cad79f2814d407182f 2024-12-09T12:15:55,536 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for e0adbf1ba01f94cad79f2814d407182f: Running coprocessor pre-open hook at 1733746555495Writing region info on filesystem at 1733746555495Initializing all the Stores at 1733746555498 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746555498Cleaning up temporary data from old regions at 1733746555513 (+15 ms)Running coprocessor post-open hooks at 1733746555536 (+23 ms)Region opened successfully at 1733746555536 2024-12-09T12:15:55,537 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1733746552866.6a763ac99f0e3a37f94a31b5f0479530. 2024-12-09T12:15:55,537 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,xxx,1733746552866.6a763ac99f0e3a37f94a31b5f0479530. 2024-12-09T12:15:55,537 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,nnn,1733746552866.fdcb3d892a0c9cf641eaedbbd9ec5937. 2024-12-09T12:15:55,537 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => fdcb3d892a0c9cf641eaedbbd9ec5937, NAME => 'test,nnn,1733746552866.fdcb3d892a0c9cf641eaedbbd9ec5937.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-12-09T12:15:55,538 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fdcb3d892a0c9cf641eaedbbd9ec5937 2024-12-09T12:15:55,538 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,nnn,1733746552866.fdcb3d892a0c9cf641eaedbbd9ec5937.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:55,538 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for fdcb3d892a0c9cf641eaedbbd9ec5937 2024-12-09T12:15:55,538 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for fdcb3d892a0c9cf641eaedbbd9ec5937 2024-12-09T12:15:55,539 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1733746552866.e0adbf1ba01f94cad79f2814d407182f., pid=46, masterSystemTime=1733746555358 2024-12-09T12:15:55,539 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=29 2024-12-09T12:15:55,540 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 7eaca6c428d8bff0cd952b3c2dfe5408, server=f15dd41c88ad,38111,1733746548467 in 484 msec 2024-12-09T12:15:55,542 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1733746552866.e0adbf1ba01f94cad79f2814d407182f. 2024-12-09T12:15:55,542 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,rrr,1733746552866.e0adbf1ba01f94cad79f2814d407182f. 2024-12-09T12:15:55,542 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,fff,1733746552866.6b5306e854c5aec62eff62a18cec5859. 2024-12-09T12:15:55,542 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 6b5306e854c5aec62eff62a18cec5859, NAME => 'test,fff,1733746552866.6b5306e854c5aec62eff62a18cec5859.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-12-09T12:15:55,543 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6b5306e854c5aec62eff62a18cec5859 2024-12-09T12:15:55,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7eaca6c428d8bff0cd952b3c2dfe5408, ASSIGN in 681 msec 2024-12-09T12:15:55,543 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,fff,1733746552866.6b5306e854c5aec62eff62a18cec5859.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:55,543 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 6b5306e854c5aec62eff62a18cec5859 2024-12-09T12:15:55,543 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 6b5306e854c5aec62eff62a18cec5859 2024-12-09T12:15:55,544 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=e0adbf1ba01f94cad79f2814d407182f, regionState=OPEN, openSeqNum=2, regionLocation=f15dd41c88ad,38111,1733746548467 2024-12-09T12:15:55,547 INFO [StoreOpener-c8f8bc12489e5c0370ba59c751783a23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c8f8bc12489e5c0370ba59c751783a23 2024-12-09T12:15:55,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0adbf1ba01f94cad79f2814d407182f, server=f15dd41c88ad,38111,1733746548467 because future has completed 2024-12-09T12:15:55,549 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=6a763ac99f0e3a37f94a31b5f0479530, regionState=OPEN, openSeqNum=2, regionLocation=f15dd41c88ad,36579,1733746548727 2024-12-09T12:15:55,550 INFO [StoreOpener-c8f8bc12489e5c0370ba59c751783a23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8f8bc12489e5c0370ba59c751783a23 columnFamilyName family 2024-12-09T12:15:55,551 DEBUG [StoreOpener-c8f8bc12489e5c0370ba59c751783a23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:55,551 INFO [StoreOpener-c8f8bc12489e5c0370ba59c751783a23-1 {}] regionserver.HStore(327): Store=c8f8bc12489e5c0370ba59c751783a23/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:55,552 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for c8f8bc12489e5c0370ba59c751783a23 2024-12-09T12:15:55,552 INFO [StoreOpener-fdcb3d892a0c9cf641eaedbbd9ec5937-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fdcb3d892a0c9cf641eaedbbd9ec5937 2024-12-09T12:15:55,552 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a763ac99f0e3a37f94a31b5f0479530, server=f15dd41c88ad,36579,1733746548727 because future has completed 2024-12-09T12:15:55,553 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/c8f8bc12489e5c0370ba59c751783a23 2024-12-09T12:15:55,554 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/c8f8bc12489e5c0370ba59c751783a23 2024-12-09T12:15:55,554 INFO [StoreOpener-fdcb3d892a0c9cf641eaedbbd9ec5937-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fdcb3d892a0c9cf641eaedbbd9ec5937 columnFamilyName family 2024-12-09T12:15:55,554 DEBUG [StoreOpener-fdcb3d892a0c9cf641eaedbbd9ec5937-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:55,555 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for c8f8bc12489e5c0370ba59c751783a23 2024-12-09T12:15:55,555 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for c8f8bc12489e5c0370ba59c751783a23 2024-12-09T12:15:55,555 INFO [StoreOpener-fdcb3d892a0c9cf641eaedbbd9ec5937-1 {}] regionserver.HStore(327): Store=fdcb3d892a0c9cf641eaedbbd9ec5937/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:55,556 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for fdcb3d892a0c9cf641eaedbbd9ec5937 2024-12-09T12:15:55,557 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/fdcb3d892a0c9cf641eaedbbd9ec5937 2024-12-09T12:15:55,558 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/fdcb3d892a0c9cf641eaedbbd9ec5937 2024-12-09T12:15:55,559 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for c8f8bc12489e5c0370ba59c751783a23 2024-12-09T12:15:55,559 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for fdcb3d892a0c9cf641eaedbbd9ec5937 2024-12-09T12:15:55,559 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for fdcb3d892a0c9cf641eaedbbd9ec5937 2024-12-09T12:15:55,559 INFO [StoreOpener-6b5306e854c5aec62eff62a18cec5859-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6b5306e854c5aec62eff62a18cec5859 2024-12-09T12:15:55,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=22 2024-12-09T12:15:55,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure e0adbf1ba01f94cad79f2814d407182f, server=f15dd41c88ad,38111,1733746548467 in 319 msec 2024-12-09T12:15:55,562 INFO [StoreOpener-6b5306e854c5aec62eff62a18cec5859-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b5306e854c5aec62eff62a18cec5859 columnFamilyName family 2024-12-09T12:15:55,562 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for fdcb3d892a0c9cf641eaedbbd9ec5937 2024-12-09T12:15:55,563 DEBUG [StoreOpener-6b5306e854c5aec62eff62a18cec5859-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:55,564 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/c8f8bc12489e5c0370ba59c751783a23/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T12:15:55,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e0adbf1ba01f94cad79f2814d407182f, ASSIGN in 703 msec 2024-12-09T12:15:55,567 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened c8f8bc12489e5c0370ba59c751783a23; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65864068, jitterRate=-0.018548905849456787}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:15:55,567 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c8f8bc12489e5c0370ba59c751783a23 2024-12-09T12:15:55,567 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for c8f8bc12489e5c0370ba59c751783a23: Running coprocessor pre-open hook at 1733746555534Writing region info on filesystem at 1733746555534Initializing all the Stores at 1733746555539 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746555540 (+1 ms)Cleaning up temporary data from old regions at 1733746555555 (+15 ms)Running coprocessor post-open hooks at 1733746555567 (+12 ms)Region opened successfully at 1733746555567 2024-12-09T12:15:55,568 INFO [StoreOpener-6b5306e854c5aec62eff62a18cec5859-1 {}] regionserver.HStore(327): Store=6b5306e854c5aec62eff62a18cec5859/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:55,570 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 6b5306e854c5aec62eff62a18cec5859 2024-12-09T12:15:55,570 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1733746552866.c8f8bc12489e5c0370ba59c751783a23., pid=54, masterSystemTime=1733746555465 2024-12-09T12:15:55,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=28 2024-12-09T12:15:55,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 6a763ac99f0e3a37f94a31b5f0479530, server=f15dd41c88ad,36579,1733746548727 in 372 msec 2024-12-09T12:15:55,576 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1733746552866.c8f8bc12489e5c0370ba59c751783a23. 2024-12-09T12:15:55,576 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,zzz,1733746552866.c8f8bc12489e5c0370ba59c751783a23. 2024-12-09T12:15:55,576 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,hhh,1733746552866.a258d2ecd9a494e8d56406b774da28ef. 2024-12-09T12:15:55,576 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => a258d2ecd9a494e8d56406b774da28ef, NAME => 'test,hhh,1733746552866.a258d2ecd9a494e8d56406b774da28ef.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-12-09T12:15:55,577 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a258d2ecd9a494e8d56406b774da28ef 2024-12-09T12:15:55,577 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,hhh,1733746552866.a258d2ecd9a494e8d56406b774da28ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:55,577 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for a258d2ecd9a494e8d56406b774da28ef 2024-12-09T12:15:55,577 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for a258d2ecd9a494e8d56406b774da28ef 2024-12-09T12:15:55,578 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/6b5306e854c5aec62eff62a18cec5859 2024-12-09T12:15:55,578 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/fdcb3d892a0c9cf641eaedbbd9ec5937/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T12:15:55,579 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened fdcb3d892a0c9cf641eaedbbd9ec5937; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70246236, jitterRate=0.04675048589706421}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:15:55,579 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/6b5306e854c5aec62eff62a18cec5859 2024-12-09T12:15:55,579 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fdcb3d892a0c9cf641eaedbbd9ec5937 2024-12-09T12:15:55,579 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for fdcb3d892a0c9cf641eaedbbd9ec5937: Running coprocessor pre-open hook at 1733746555538Writing region info on filesystem at 1733746555538Initializing all the Stores at 1733746555541 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746555541Cleaning up temporary data from old regions at 1733746555559 (+18 ms)Running coprocessor post-open hooks at 1733746555579 (+20 ms)Region opened successfully at 1733746555579 2024-12-09T12:15:55,581 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 6b5306e854c5aec62eff62a18cec5859 2024-12-09T12:15:55,581 INFO [StoreOpener-a258d2ecd9a494e8d56406b774da28ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a258d2ecd9a494e8d56406b774da28ef 2024-12-09T12:15:55,581 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1733746552866.fdcb3d892a0c9cf641eaedbbd9ec5937., pid=38, masterSystemTime=1733746555291 2024-12-09T12:15:55,581 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 6b5306e854c5aec62eff62a18cec5859 2024-12-09T12:15:55,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6a763ac99f0e3a37f94a31b5f0479530, ASSIGN in 716 msec 2024-12-09T12:15:55,584 INFO [StoreOpener-a258d2ecd9a494e8d56406b774da28ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a258d2ecd9a494e8d56406b774da28ef columnFamilyName family 2024-12-09T12:15:55,585 DEBUG [StoreOpener-a258d2ecd9a494e8d56406b774da28ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:55,585 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 6b5306e854c5aec62eff62a18cec5859 2024-12-09T12:15:55,585 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=c8f8bc12489e5c0370ba59c751783a23, regionState=OPEN, openSeqNum=2, regionLocation=f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:55,585 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1733746552866.fdcb3d892a0c9cf641eaedbbd9ec5937. 2024-12-09T12:15:55,586 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,nnn,1733746552866.fdcb3d892a0c9cf641eaedbbd9ec5937. 2024-12-09T12:15:55,586 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,sss,1733746552866.603cf0cbb0e068a5f7727e4abab3a5d5. 2024-12-09T12:15:55,586 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 603cf0cbb0e068a5f7727e4abab3a5d5, NAME => 'test,sss,1733746552866.603cf0cbb0e068a5f7727e4abab3a5d5.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-12-09T12:15:55,586 INFO [StoreOpener-a258d2ecd9a494e8d56406b774da28ef-1 {}] regionserver.HStore(327): Store=a258d2ecd9a494e8d56406b774da28ef/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:55,586 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 603cf0cbb0e068a5f7727e4abab3a5d5 2024-12-09T12:15:55,587 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,sss,1733746552866.603cf0cbb0e068a5f7727e4abab3a5d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:55,587 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for a258d2ecd9a494e8d56406b774da28ef 2024-12-09T12:15:55,587 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 603cf0cbb0e068a5f7727e4abab3a5d5 2024-12-09T12:15:55,587 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 603cf0cbb0e068a5f7727e4abab3a5d5 2024-12-09T12:15:55,588 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/a258d2ecd9a494e8d56406b774da28ef 2024-12-09T12:15:55,588 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/a258d2ecd9a494e8d56406b774da28ef 2024-12-09T12:15:55,590 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for a258d2ecd9a494e8d56406b774da28ef 2024-12-09T12:15:55,590 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for a258d2ecd9a494e8d56406b774da28ef 2024-12-09T12:15:55,592 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8f8bc12489e5c0370ba59c751783a23, server=f15dd41c88ad,33783,1733746548634 because future has completed 2024-12-09T12:15:55,595 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for a258d2ecd9a494e8d56406b774da28ef 2024-12-09T12:15:55,596 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/6b5306e854c5aec62eff62a18cec5859/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T12:15:55,598 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 6b5306e854c5aec62eff62a18cec5859; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72415676, jitterRate=0.07907766103744507}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:15:55,598 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6b5306e854c5aec62eff62a18cec5859 2024-12-09T12:15:55,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=30 2024-12-09T12:15:55,598 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 6b5306e854c5aec62eff62a18cec5859: Running coprocessor pre-open hook at 1733746555543Writing region info on filesystem at 1733746555543Initializing all the Stores at 1733746555545 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746555545Cleaning up temporary data from old regions at 1733746555581 (+36 ms)Running coprocessor post-open hooks at 1733746555598 (+17 ms)Region opened successfully at 1733746555598 2024-12-09T12:15:55,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure c8f8bc12489e5c0370ba59c751783a23, server=f15dd41c88ad,33783,1733746548634 in 223 msec 2024-12-09T12:15:55,600 INFO [StoreOpener-603cf0cbb0e068a5f7727e4abab3a5d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 603cf0cbb0e068a5f7727e4abab3a5d5 2024-12-09T12:15:55,602 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN in 739 msec 2024-12-09T12:15:55,602 INFO [StoreOpener-603cf0cbb0e068a5f7727e4abab3a5d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 603cf0cbb0e068a5f7727e4abab3a5d5 columnFamilyName family 2024-12-09T12:15:55,602 DEBUG [StoreOpener-603cf0cbb0e068a5f7727e4abab3a5d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:55,602 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1733746552866.6b5306e854c5aec62eff62a18cec5859., pid=43, masterSystemTime=1733746555358 2024-12-09T12:15:55,603 INFO [StoreOpener-603cf0cbb0e068a5f7727e4abab3a5d5-1 {}] regionserver.HStore(327): Store=603cf0cbb0e068a5f7727e4abab3a5d5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:55,603 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 603cf0cbb0e068a5f7727e4abab3a5d5 2024-12-09T12:15:55,604 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/603cf0cbb0e068a5f7727e4abab3a5d5 2024-12-09T12:15:55,605 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/603cf0cbb0e068a5f7727e4abab3a5d5 2024-12-09T12:15:55,605 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 603cf0cbb0e068a5f7727e4abab3a5d5 2024-12-09T12:15:55,605 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 603cf0cbb0e068a5f7727e4abab3a5d5 2024-12-09T12:15:55,613 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 603cf0cbb0e068a5f7727e4abab3a5d5 2024-12-09T12:15:55,614 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/a258d2ecd9a494e8d56406b774da28ef/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T12:15:55,615 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened a258d2ecd9a494e8d56406b774da28ef; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74725457, jitterRate=0.1134960800409317}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:15:55,615 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a258d2ecd9a494e8d56406b774da28ef 2024-12-09T12:15:55,615 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for a258d2ecd9a494e8d56406b774da28ef: Running coprocessor pre-open hook at 1733746555577Writing region info on filesystem at 1733746555577Initializing all the Stores at 1733746555579 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746555579Cleaning up temporary data from old regions at 1733746555590 (+11 ms)Running coprocessor post-open hooks at 1733746555615 (+25 ms)Region opened successfully at 1733746555615 2024-12-09T12:15:55,618 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1733746552866.a258d2ecd9a494e8d56406b774da28ef., pid=50, masterSystemTime=1733746555465 2024-12-09T12:15:55,627 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/603cf0cbb0e068a5f7727e4abab3a5d5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T12:15:55,628 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=fdcb3d892a0c9cf641eaedbbd9ec5937, regionState=OPEN, openSeqNum=2, regionLocation=f15dd41c88ad,36579,1733746548727 2024-12-09T12:15:55,629 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1733746552866.6b5306e854c5aec62eff62a18cec5859. 2024-12-09T12:15:55,629 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,fff,1733746552866.6b5306e854c5aec62eff62a18cec5859. 2024-12-09T12:15:55,630 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,iii,1733746552866.a0cf25c9917186a9e3dbd87e44f33615. 2024-12-09T12:15:55,630 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => a0cf25c9917186a9e3dbd87e44f33615, NAME => 'test,iii,1733746552866.a0cf25c9917186a9e3dbd87e44f33615.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-12-09T12:15:55,630 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 603cf0cbb0e068a5f7727e4abab3a5d5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72628752, jitterRate=0.08225274085998535}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:15:55,630 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 603cf0cbb0e068a5f7727e4abab3a5d5 2024-12-09T12:15:55,630 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 603cf0cbb0e068a5f7727e4abab3a5d5: Running coprocessor pre-open hook at 1733746555587Writing region info on filesystem at 1733746555587Initializing all the Stores at 1733746555593 (+6 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746555593Cleaning up temporary data from old regions at 1733746555605 (+12 ms)Running coprocessor post-open hooks at 1733746555630 (+25 ms)Region opened successfully at 1733746555630 2024-12-09T12:15:55,630 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a0cf25c9917186a9e3dbd87e44f33615 2024-12-09T12:15:55,630 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,iii,1733746552866.a0cf25c9917186a9e3dbd87e44f33615.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:55,631 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for a0cf25c9917186a9e3dbd87e44f33615 2024-12-09T12:15:55,631 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for a0cf25c9917186a9e3dbd87e44f33615 2024-12-09T12:15:55,632 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1733746552866.a258d2ecd9a494e8d56406b774da28ef. 2024-12-09T12:15:55,632 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,hhh,1733746552866.a258d2ecd9a494e8d56406b774da28ef. 2024-12-09T12:15:55,632 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,lll,1733746552866.ebee1500f62faadfac8720740a1e527e. 2024-12-09T12:15:55,632 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1733746552866.603cf0cbb0e068a5f7727e4abab3a5d5., pid=40, masterSystemTime=1733746555291 2024-12-09T12:15:55,632 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => ebee1500f62faadfac8720740a1e527e, NAME => 'test,lll,1733746552866.ebee1500f62faadfac8720740a1e527e.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-12-09T12:15:55,632 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ebee1500f62faadfac8720740a1e527e 2024-12-09T12:15:55,633 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,lll,1733746552866.ebee1500f62faadfac8720740a1e527e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:55,633 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for ebee1500f62faadfac8720740a1e527e 2024-12-09T12:15:55,633 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for ebee1500f62faadfac8720740a1e527e 2024-12-09T12:15:55,635 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=38, ppid=18, state=RUNNABLE, hasLock=true; OpenRegionProcedure fdcb3d892a0c9cf641eaedbbd9ec5937, server=f15dd41c88ad,36579,1733746548727 2024-12-09T12:15:55,636 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1733746552866.603cf0cbb0e068a5f7727e4abab3a5d5. 2024-12-09T12:15:55,636 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,sss,1733746552866.603cf0cbb0e068a5f7727e4abab3a5d5. 2024-12-09T12:15:55,636 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,uuu,1733746552866.b0351c7e96a16dae18ff5a8d1664b2b1. 2024-12-09T12:15:55,637 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => b0351c7e96a16dae18ff5a8d1664b2b1, NAME => 'test,uuu,1733746552866.b0351c7e96a16dae18ff5a8d1664b2b1.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-12-09T12:15:55,637 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b0351c7e96a16dae18ff5a8d1664b2b1 2024-12-09T12:15:55,637 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,uuu,1733746552866.b0351c7e96a16dae18ff5a8d1664b2b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:55,637 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for b0351c7e96a16dae18ff5a8d1664b2b1 2024-12-09T12:15:55,637 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for b0351c7e96a16dae18ff5a8d1664b2b1 2024-12-09T12:15:55,640 INFO [StoreOpener-a0cf25c9917186a9e3dbd87e44f33615-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a0cf25c9917186a9e3dbd87e44f33615 2024-12-09T12:15:55,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=18 2024-12-09T12:15:55,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure fdcb3d892a0c9cf641eaedbbd9ec5937, server=f15dd41c88ad,36579,1733746548727 in 476 msec 2024-12-09T12:15:55,643 INFO [StoreOpener-a0cf25c9917186a9e3dbd87e44f33615-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0cf25c9917186a9e3dbd87e44f33615 columnFamilyName family 2024-12-09T12:15:55,644 DEBUG [StoreOpener-a0cf25c9917186a9e3dbd87e44f33615-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:55,645 INFO [StoreOpener-a0cf25c9917186a9e3dbd87e44f33615-1 {}] regionserver.HStore(327): Store=a0cf25c9917186a9e3dbd87e44f33615/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:55,645 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for a0cf25c9917186a9e3dbd87e44f33615 2024-12-09T12:15:55,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fdcb3d892a0c9cf641eaedbbd9ec5937, ASSIGN in 783 msec 2024-12-09T12:15:55,646 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/a0cf25c9917186a9e3dbd87e44f33615 2024-12-09T12:15:55,646 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=603cf0cbb0e068a5f7727e4abab3a5d5, regionState=OPEN, openSeqNum=2, regionLocation=f15dd41c88ad,36579,1733746548727 2024-12-09T12:15:55,647 INFO [StoreOpener-ebee1500f62faadfac8720740a1e527e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ebee1500f62faadfac8720740a1e527e 2024-12-09T12:15:55,647 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/a0cf25c9917186a9e3dbd87e44f33615 2024-12-09T12:15:55,648 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for a0cf25c9917186a9e3dbd87e44f33615 2024-12-09T12:15:55,648 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for a0cf25c9917186a9e3dbd87e44f33615 2024-12-09T12:15:55,651 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for a0cf25c9917186a9e3dbd87e44f33615 2024-12-09T12:15:55,655 INFO [StoreOpener-b0351c7e96a16dae18ff5a8d1664b2b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b0351c7e96a16dae18ff5a8d1664b2b1 2024-12-09T12:15:55,655 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/a0cf25c9917186a9e3dbd87e44f33615/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T12:15:55,656 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 603cf0cbb0e068a5f7727e4abab3a5d5, server=f15dd41c88ad,36579,1733746548727 because future has completed 2024-12-09T12:15:55,656 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened a0cf25c9917186a9e3dbd87e44f33615; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75146283, jitterRate=0.11976687610149384}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:15:55,656 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a0cf25c9917186a9e3dbd87e44f33615 2024-12-09T12:15:55,657 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for a0cf25c9917186a9e3dbd87e44f33615: Running coprocessor pre-open hook at 1733746555631Writing region info on filesystem at 1733746555631Initializing all the Stores at 1733746555634 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746555634Cleaning up temporary data from old regions at 1733746555648 (+14 ms)Running coprocessor post-open hooks at 1733746555656 (+8 ms)Region opened successfully at 1733746555656 2024-12-09T12:15:55,657 INFO [StoreOpener-ebee1500f62faadfac8720740a1e527e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ebee1500f62faadfac8720740a1e527e columnFamilyName family 2024-12-09T12:15:55,657 DEBUG [StoreOpener-ebee1500f62faadfac8720740a1e527e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:55,658 INFO [StoreOpener-ebee1500f62faadfac8720740a1e527e-1 {}] regionserver.HStore(327): Store=ebee1500f62faadfac8720740a1e527e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:55,658 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for ebee1500f62faadfac8720740a1e527e 2024-12-09T12:15:55,658 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1733746552866.a0cf25c9917186a9e3dbd87e44f33615., pid=44, masterSystemTime=1733746555358 2024-12-09T12:15:55,659 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/ebee1500f62faadfac8720740a1e527e 2024-12-09T12:15:55,660 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/ebee1500f62faadfac8720740a1e527e 2024-12-09T12:15:55,661 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for ebee1500f62faadfac8720740a1e527e 2024-12-09T12:15:55,661 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for ebee1500f62faadfac8720740a1e527e 2024-12-09T12:15:55,662 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1733746552866.a0cf25c9917186a9e3dbd87e44f33615. 2024-12-09T12:15:55,662 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,iii,1733746552866.a0cf25c9917186a9e3dbd87e44f33615. 2024-12-09T12:15:55,662 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,jjj,1733746552866.d9e23742363329565223433d05d86296. 2024-12-09T12:15:55,662 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => d9e23742363329565223433d05d86296, NAME => 'test,jjj,1733746552866.d9e23742363329565223433d05d86296.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-12-09T12:15:55,663 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d9e23742363329565223433d05d86296 2024-12-09T12:15:55,663 INFO [StoreOpener-b0351c7e96a16dae18ff5a8d1664b2b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b0351c7e96a16dae18ff5a8d1664b2b1 columnFamilyName family 2024-12-09T12:15:55,663 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,jjj,1733746552866.d9e23742363329565223433d05d86296.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:55,663 DEBUG [StoreOpener-b0351c7e96a16dae18ff5a8d1664b2b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:55,663 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for d9e23742363329565223433d05d86296 2024-12-09T12:15:55,663 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for d9e23742363329565223433d05d86296 2024-12-09T12:15:55,665 INFO [StoreOpener-b0351c7e96a16dae18ff5a8d1664b2b1-1 {}] regionserver.HStore(327): Store=b0351c7e96a16dae18ff5a8d1664b2b1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:55,665 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for b0351c7e96a16dae18ff5a8d1664b2b1 2024-12-09T12:15:55,666 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for ebee1500f62faadfac8720740a1e527e 2024-12-09T12:15:55,666 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/b0351c7e96a16dae18ff5a8d1664b2b1 2024-12-09T12:15:55,667 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/b0351c7e96a16dae18ff5a8d1664b2b1 2024-12-09T12:15:55,667 INFO [StoreOpener-d9e23742363329565223433d05d86296-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d9e23742363329565223433d05d86296 2024-12-09T12:15:55,668 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for b0351c7e96a16dae18ff5a8d1664b2b1 2024-12-09T12:15:55,668 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for b0351c7e96a16dae18ff5a8d1664b2b1 2024-12-09T12:15:55,670 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=23 2024-12-09T12:15:55,670 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 603cf0cbb0e068a5f7727e4abab3a5d5, server=f15dd41c88ad,36579,1733746548727 in 480 msec 2024-12-09T12:15:55,670 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for b0351c7e96a16dae18ff5a8d1664b2b1 2024-12-09T12:15:55,670 INFO [StoreOpener-d9e23742363329565223433d05d86296-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9e23742363329565223433d05d86296 columnFamilyName family 2024-12-09T12:15:55,671 DEBUG [StoreOpener-d9e23742363329565223433d05d86296-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:55,672 INFO [StoreOpener-d9e23742363329565223433d05d86296-1 {}] regionserver.HStore(327): Store=d9e23742363329565223433d05d86296/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:55,672 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for d9e23742363329565223433d05d86296 2024-12-09T12:15:55,673 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/d9e23742363329565223433d05d86296 2024-12-09T12:15:55,674 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/d9e23742363329565223433d05d86296 2024-12-09T12:15:55,675 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for d9e23742363329565223433d05d86296 2024-12-09T12:15:55,675 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for d9e23742363329565223433d05d86296 2024-12-09T12:15:55,675 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/ebee1500f62faadfac8720740a1e527e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T12:15:55,676 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened ebee1500f62faadfac8720740a1e527e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60968333, jitterRate=-0.09150104224681854}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:15:55,676 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ebee1500f62faadfac8720740a1e527e 2024-12-09T12:15:55,676 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for ebee1500f62faadfac8720740a1e527e: Running coprocessor pre-open hook at 1733746555633Writing region info on filesystem at 1733746555633Initializing all the Stores at 1733746555635 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746555635Cleaning up temporary data from old regions at 1733746555661 (+26 ms)Running coprocessor post-open hooks at 1733746555676 (+15 ms)Region opened successfully at 1733746555676 2024-12-09T12:15:55,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=603cf0cbb0e068a5f7727e4abab3a5d5, ASSIGN in 811 msec 2024-12-09T12:15:55,677 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1733746552866.ebee1500f62faadfac8720740a1e527e., pid=51, masterSystemTime=1733746555465 2024-12-09T12:15:55,679 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for d9e23742363329565223433d05d86296 2024-12-09T12:15:55,680 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=a0cf25c9917186a9e3dbd87e44f33615, regionState=OPEN, openSeqNum=2, regionLocation=f15dd41c88ad,38111,1733746548467 2024-12-09T12:15:55,681 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1733746552866.ebee1500f62faadfac8720740a1e527e. 2024-12-09T12:15:55,681 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,lll,1733746552866.ebee1500f62faadfac8720740a1e527e. 2024-12-09T12:15:55,681 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,ttt,1733746552866.ba4a26e129719340e5696ac56c6b61ac. 2024-12-09T12:15:55,682 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => ba4a26e129719340e5696ac56c6b61ac, NAME => 'test,ttt,1733746552866.ba4a26e129719340e5696ac56c6b61ac.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-12-09T12:15:55,682 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ba4a26e129719340e5696ac56c6b61ac 2024-12-09T12:15:55,682 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,ttt,1733746552866.ba4a26e129719340e5696ac56c6b61ac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:55,683 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for ba4a26e129719340e5696ac56c6b61ac 2024-12-09T12:15:55,683 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for ba4a26e129719340e5696ac56c6b61ac 2024-12-09T12:15:55,684 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0cf25c9917186a9e3dbd87e44f33615, server=f15dd41c88ad,38111,1733746548467 because future has completed 2024-12-09T12:15:55,684 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/b0351c7e96a16dae18ff5a8d1664b2b1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T12:15:55,685 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=ebee1500f62faadfac8720740a1e527e, regionState=OPEN, openSeqNum=2, regionLocation=f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:55,685 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/d9e23742363329565223433d05d86296/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T12:15:55,685 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened b0351c7e96a16dae18ff5a8d1664b2b1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64414149, jitterRate=-0.04015438258647919}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:15:55,686 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b0351c7e96a16dae18ff5a8d1664b2b1 2024-12-09T12:15:55,686 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for b0351c7e96a16dae18ff5a8d1664b2b1: Running coprocessor pre-open hook at 1733746555637Writing region info on filesystem at 1733746555637Initializing all the Stores at 1733746555639 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746555639Cleaning up temporary data from old regions at 1733746555668 (+29 ms)Running coprocessor post-open hooks at 1733746555686 (+18 ms)Region opened successfully at 1733746555686 2024-12-09T12:15:55,686 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened d9e23742363329565223433d05d86296; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69603996, jitterRate=0.03718036413192749}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:15:55,686 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d9e23742363329565223433d05d86296 2024-12-09T12:15:55,686 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for d9e23742363329565223433d05d86296: Running coprocessor pre-open hook at 1733746555663Writing region info on filesystem at 1733746555663Initializing all the Stores at 1733746555667 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746555667Cleaning up temporary data from old regions at 1733746555675 (+8 ms)Running coprocessor post-open hooks at 1733746555686 (+11 ms)Region opened successfully at 1733746555686 2024-12-09T12:15:55,688 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1733746552866.d9e23742363329565223433d05d86296., pid=45, masterSystemTime=1733746555358 2024-12-09T12:15:55,689 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1733746552866.b0351c7e96a16dae18ff5a8d1664b2b1., pid=41, masterSystemTime=1733746555291 2024-12-09T12:15:55,689 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure ebee1500f62faadfac8720740a1e527e, server=f15dd41c88ad,33783,1733746548634 because future has completed 2024-12-09T12:15:55,691 INFO [StoreOpener-ba4a26e129719340e5696ac56c6b61ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ba4a26e129719340e5696ac56c6b61ac 2024-12-09T12:15:55,692 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1733746552866.d9e23742363329565223433d05d86296. 2024-12-09T12:15:55,692 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,jjj,1733746552866.d9e23742363329565223433d05d86296. 2024-12-09T12:15:55,692 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,vvv,1733746552866.23280f1743db631eaa35bf3487e69147. 2024-12-09T12:15:55,692 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 23280f1743db631eaa35bf3487e69147, NAME => 'test,vvv,1733746552866.23280f1743db631eaa35bf3487e69147.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-12-09T12:15:55,693 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 23280f1743db631eaa35bf3487e69147 2024-12-09T12:15:55,693 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,vvv,1733746552866.23280f1743db631eaa35bf3487e69147.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:55,693 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 23280f1743db631eaa35bf3487e69147 2024-12-09T12:15:55,693 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 23280f1743db631eaa35bf3487e69147 2024-12-09T12:15:55,693 INFO [StoreOpener-ba4a26e129719340e5696ac56c6b61ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba4a26e129719340e5696ac56c6b61ac columnFamilyName family 2024-12-09T12:15:55,693 DEBUG [StoreOpener-ba4a26e129719340e5696ac56c6b61ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:55,694 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1733746552866.b0351c7e96a16dae18ff5a8d1664b2b1. 2024-12-09T12:15:55,694 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,uuu,1733746552866.b0351c7e96a16dae18ff5a8d1664b2b1. 2024-12-09T12:15:55,694 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,bbb,1733746552866.9ac6650f2a35b2c131e93ead260da106. 2024-12-09T12:15:55,694 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 9ac6650f2a35b2c131e93ead260da106, NAME => 'test,bbb,1733746552866.9ac6650f2a35b2c131e93ead260da106.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-12-09T12:15:55,695 INFO [StoreOpener-ba4a26e129719340e5696ac56c6b61ac-1 {}] regionserver.HStore(327): Store=ba4a26e129719340e5696ac56c6b61ac/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:55,696 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9ac6650f2a35b2c131e93ead260da106 2024-12-09T12:15:55,696 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for ba4a26e129719340e5696ac56c6b61ac 2024-12-09T12:15:55,696 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,bbb,1733746552866.9ac6650f2a35b2c131e93ead260da106.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:55,696 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 9ac6650f2a35b2c131e93ead260da106 2024-12-09T12:15:55,696 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 9ac6650f2a35b2c131e93ead260da106 2024-12-09T12:15:55,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=13 2024-12-09T12:15:55,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure a0cf25c9917186a9e3dbd87e44f33615, server=f15dd41c88ad,38111,1733746548467 in 477 msec 2024-12-09T12:15:55,698 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/ba4a26e129719340e5696ac56c6b61ac 2024-12-09T12:15:55,699 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/ba4a26e129719340e5696ac56c6b61ac 2024-12-09T12:15:55,699 INFO [StoreOpener-23280f1743db631eaa35bf3487e69147-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 23280f1743db631eaa35bf3487e69147 2024-12-09T12:15:55,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a0cf25c9917186a9e3dbd87e44f33615, ASSIGN in 838 msec 2024-12-09T12:15:55,699 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for ba4a26e129719340e5696ac56c6b61ac 2024-12-09T12:15:55,699 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for ba4a26e129719340e5696ac56c6b61ac 2024-12-09T12:15:55,700 INFO [StoreOpener-9ac6650f2a35b2c131e93ead260da106-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9ac6650f2a35b2c131e93ead260da106 2024-12-09T12:15:55,701 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=b0351c7e96a16dae18ff5a8d1664b2b1, regionState=OPEN, openSeqNum=2, regionLocation=f15dd41c88ad,36579,1733746548727 2024-12-09T12:15:55,704 INFO [StoreOpener-9ac6650f2a35b2c131e93ead260da106-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9ac6650f2a35b2c131e93ead260da106 columnFamilyName family 2024-12-09T12:15:55,704 DEBUG [StoreOpener-9ac6650f2a35b2c131e93ead260da106-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:55,704 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for ba4a26e129719340e5696ac56c6b61ac 2024-12-09T12:15:55,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure b0351c7e96a16dae18ff5a8d1664b2b1, server=f15dd41c88ad,36579,1733746548727 because future has completed 2024-12-09T12:15:55,705 INFO [StoreOpener-23280f1743db631eaa35bf3487e69147-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23280f1743db631eaa35bf3487e69147 columnFamilyName family 2024-12-09T12:15:55,705 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=d9e23742363329565223433d05d86296, regionState=OPEN, openSeqNum=2, regionLocation=f15dd41c88ad,38111,1733746548467 2024-12-09T12:15:55,705 DEBUG [StoreOpener-23280f1743db631eaa35bf3487e69147-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:55,706 INFO [StoreOpener-9ac6650f2a35b2c131e93ead260da106-1 {}] regionserver.HStore(327): Store=9ac6650f2a35b2c131e93ead260da106/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:55,706 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 9ac6650f2a35b2c131e93ead260da106 2024-12-09T12:15:55,707 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/9ac6650f2a35b2c131e93ead260da106 2024-12-09T12:15:55,708 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/9ac6650f2a35b2c131e93ead260da106 2024-12-09T12:15:55,708 INFO [StoreOpener-23280f1743db631eaa35bf3487e69147-1 {}] regionserver.HStore(327): Store=23280f1743db631eaa35bf3487e69147/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:55,708 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 23280f1743db631eaa35bf3487e69147 2024-12-09T12:15:55,709 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 9ac6650f2a35b2c131e93ead260da106 2024-12-09T12:15:55,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure d9e23742363329565223433d05d86296, server=f15dd41c88ad,38111,1733746548467 because future has completed 2024-12-09T12:15:55,709 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 9ac6650f2a35b2c131e93ead260da106 2024-12-09T12:15:55,709 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/23280f1743db631eaa35bf3487e69147 2024-12-09T12:15:55,710 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/23280f1743db631eaa35bf3487e69147 2024-12-09T12:15:55,711 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 23280f1743db631eaa35bf3487e69147 2024-12-09T12:15:55,711 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 23280f1743db631eaa35bf3487e69147 2024-12-09T12:15:55,712 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 9ac6650f2a35b2c131e93ead260da106 2024-12-09T12:15:55,715 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=25 2024-12-09T12:15:55,715 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure b0351c7e96a16dae18ff5a8d1664b2b1, server=f15dd41c88ad,36579,1733746548727 in 519 msec 2024-12-09T12:15:55,716 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 23280f1743db631eaa35bf3487e69147 2024-12-09T12:15:55,721 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/ba4a26e129719340e5696ac56c6b61ac/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T12:15:55,722 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened ba4a26e129719340e5696ac56c6b61ac; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62163013, jitterRate=-0.07369892299175262}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:15:55,722 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ba4a26e129719340e5696ac56c6b61ac 2024-12-09T12:15:55,722 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for ba4a26e129719340e5696ac56c6b61ac: Running coprocessor pre-open hook at 1733746555683Writing region info on filesystem at 1733746555683Initializing all the Stores at 1733746555687 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746555687Cleaning up temporary data from old regions at 1733746555699 (+12 ms)Running coprocessor post-open hooks at 1733746555722 (+23 ms)Region opened successfully at 1733746555722 2024-12-09T12:15:55,724 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/23280f1743db631eaa35bf3487e69147/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T12:15:55,724 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1733746552866.ba4a26e129719340e5696ac56c6b61ac., pid=53, masterSystemTime=1733746555465 2024-12-09T12:15:55,726 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 23280f1743db631eaa35bf3487e69147; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73677713, jitterRate=0.09788347780704498}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:15:55,726 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 23280f1743db631eaa35bf3487e69147 2024-12-09T12:15:55,726 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 23280f1743db631eaa35bf3487e69147: Running coprocessor pre-open hook at 1733746555693Writing region info on filesystem at 1733746555693Initializing all the Stores at 1733746555696 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746555696Cleaning up temporary data from old regions at 1733746555712 (+16 ms)Running coprocessor post-open hooks at 1733746555726 (+14 ms)Region opened successfully at 1733746555726 2024-12-09T12:15:55,728 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1733746552866.23280f1743db631eaa35bf3487e69147., pid=47, masterSystemTime=1733746555358 2024-12-09T12:15:55,731 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/9ac6650f2a35b2c131e93ead260da106/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T12:15:55,732 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1733746552866.ba4a26e129719340e5696ac56c6b61ac. 2024-12-09T12:15:55,732 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,ttt,1733746552866.ba4a26e129719340e5696ac56c6b61ac. 2024-12-09T12:15:55,732 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21. 2024-12-09T12:15:55,732 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 9ac6650f2a35b2c131e93ead260da106; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59675268, jitterRate=-0.11076921224594116}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:15:55,732 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9ac6650f2a35b2c131e93ead260da106 2024-12-09T12:15:55,732 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => e5251eee3971dde9f0f276cfd8632c21, NAME => 'test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-12-09T12:15:55,733 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e5251eee3971dde9f0f276cfd8632c21 2024-12-09T12:15:55,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b0351c7e96a16dae18ff5a8d1664b2b1, ASSIGN in 856 msec 2024-12-09T12:15:55,733 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:55,733 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for e5251eee3971dde9f0f276cfd8632c21 2024-12-09T12:15:55,733 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for e5251eee3971dde9f0f276cfd8632c21 2024-12-09T12:15:55,733 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 9ac6650f2a35b2c131e93ead260da106: Running coprocessor pre-open hook at 1733746555696Writing region info on filesystem at 1733746555696Initializing all the Stores at 1733746555699 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746555699Cleaning up temporary data from old regions at 1733746555709 (+10 ms)Running coprocessor post-open hooks at 1733746555732 (+23 ms)Region opened successfully at 1733746555732 2024-12-09T12:15:55,736 INFO [StoreOpener-e5251eee3971dde9f0f276cfd8632c21-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e5251eee3971dde9f0f276cfd8632c21 2024-12-09T12:15:55,736 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1733746552866.9ac6650f2a35b2c131e93ead260da106., pid=35, masterSystemTime=1733746555291 2024-12-09T12:15:55,737 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1733746552866.23280f1743db631eaa35bf3487e69147. 2024-12-09T12:15:55,737 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,vvv,1733746552866.23280f1743db631eaa35bf3487e69147. 2024-12-09T12:15:55,739 INFO [StoreOpener-e5251eee3971dde9f0f276cfd8632c21-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5251eee3971dde9f0f276cfd8632c21 columnFamilyName family 2024-12-09T12:15:55,740 DEBUG [StoreOpener-e5251eee3971dde9f0f276cfd8632c21-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:55,740 INFO [StoreOpener-e5251eee3971dde9f0f276cfd8632c21-1 {}] regionserver.HStore(327): Store=e5251eee3971dde9f0f276cfd8632c21/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:55,741 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for e5251eee3971dde9f0f276cfd8632c21 2024-12-09T12:15:55,743 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=ba4a26e129719340e5696ac56c6b61ac, regionState=OPEN, openSeqNum=2, regionLocation=f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:55,744 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1733746552866.9ac6650f2a35b2c131e93ead260da106. 2024-12-09T12:15:55,744 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,bbb,1733746552866.9ac6650f2a35b2c131e93ead260da106. 2024-12-09T12:15:55,744 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,ggg,1733746552866.c910d27fbf3265165c7fd59609e57c82. 2024-12-09T12:15:55,745 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/e5251eee3971dde9f0f276cfd8632c21 2024-12-09T12:15:55,745 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => c910d27fbf3265165c7fd59609e57c82, NAME => 'test,ggg,1733746552866.c910d27fbf3265165c7fd59609e57c82.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-12-09T12:15:55,745 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c910d27fbf3265165c7fd59609e57c82 2024-12-09T12:15:55,745 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,ggg,1733746552866.c910d27fbf3265165c7fd59609e57c82.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:55,745 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for c910d27fbf3265165c7fd59609e57c82 2024-12-09T12:15:55,745 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for c910d27fbf3265165c7fd59609e57c82 2024-12-09T12:15:55,746 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f15dd41c88ad,33783,1733746548634, table=test, region=ebee1500f62faadfac8720740a1e527e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-09T12:15:55,746 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f15dd41c88ad,33783,1733746548634, table=test, region=ba4a26e129719340e5696ac56c6b61ac. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-09T12:15:55,746 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/e5251eee3971dde9f0f276cfd8632c21 2024-12-09T12:15:55,747 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba4a26e129719340e5696ac56c6b61ac, server=f15dd41c88ad,33783,1733746548634 because future has completed 2024-12-09T12:15:55,751 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for e5251eee3971dde9f0f276cfd8632c21 2024-12-09T12:15:55,751 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for e5251eee3971dde9f0f276cfd8632c21 2024-12-09T12:15:55,755 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=9ac6650f2a35b2c131e93ead260da106, regionState=OPEN, openSeqNum=2, regionLocation=f15dd41c88ad,36579,1733746548727 2024-12-09T12:15:55,755 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for e5251eee3971dde9f0f276cfd8632c21 2024-12-09T12:15:55,758 INFO [StoreOpener-c910d27fbf3265165c7fd59609e57c82-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c910d27fbf3265165c7fd59609e57c82 2024-12-09T12:15:55,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ac6650f2a35b2c131e93ead260da106, server=f15dd41c88ad,36579,1733746548727 because future has completed 2024-12-09T12:15:55,767 INFO [StoreOpener-c910d27fbf3265165c7fd59609e57c82-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c910d27fbf3265165c7fd59609e57c82 columnFamilyName family 2024-12-09T12:15:55,767 DEBUG [StoreOpener-c910d27fbf3265165c7fd59609e57c82-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:55,771 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/e5251eee3971dde9f0f276cfd8632c21/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T12:15:55,773 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened e5251eee3971dde9f0f276cfd8632c21; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67049801, jitterRate=-8.801072835922241E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:15:55,773 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e5251eee3971dde9f0f276cfd8632c21 2024-12-09T12:15:55,773 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for e5251eee3971dde9f0f276cfd8632c21: Running coprocessor pre-open hook at 1733746555733Writing region info on filesystem at 1733746555733Initializing all the Stores at 1733746555735 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746555735Cleaning up temporary data from old regions at 1733746555751 (+16 ms)Running coprocessor post-open hooks at 1733746555773 (+22 ms)Region opened successfully at 1733746555773 2024-12-09T12:15:55,774 INFO [StoreOpener-c910d27fbf3265165c7fd59609e57c82-1 {}] regionserver.HStore(327): Store=c910d27fbf3265165c7fd59609e57c82/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:55,775 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for c910d27fbf3265165c7fd59609e57c82 2024-12-09T12:15:55,776 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/c910d27fbf3265165c7fd59609e57c82 2024-12-09T12:15:55,776 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/c910d27fbf3265165c7fd59609e57c82 2024-12-09T12:15:55,777 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for c910d27fbf3265165c7fd59609e57c82 2024-12-09T12:15:55,777 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for c910d27fbf3265165c7fd59609e57c82 2024-12-09T12:15:55,779 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21., pid=55, masterSystemTime=1733746555465 2024-12-09T12:15:55,781 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for c910d27fbf3265165c7fd59609e57c82 2024-12-09T12:15:55,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=24 2024-12-09T12:15:55,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure ba4a26e129719340e5696ac56c6b61ac, server=f15dd41c88ad,33783,1733746548634 in 407 msec 2024-12-09T12:15:55,783 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21. 2024-12-09T12:15:55,783 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21. 2024-12-09T12:15:55,783 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,mmm,1733746552866.62a60a0dff0c026a110e755e6c249001. 2024-12-09T12:15:55,784 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 62a60a0dff0c026a110e755e6c249001, NAME => 'test,mmm,1733746552866.62a60a0dff0c026a110e755e6c249001.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-12-09T12:15:55,784 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 62a60a0dff0c026a110e755e6c249001 2024-12-09T12:15:55,784 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,mmm,1733746552866.62a60a0dff0c026a110e755e6c249001.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:55,784 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 62a60a0dff0c026a110e755e6c249001 2024-12-09T12:15:55,784 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 62a60a0dff0c026a110e755e6c249001 2024-12-09T12:15:55,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN in 923 msec 2024-12-09T12:15:55,787 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/c910d27fbf3265165c7fd59609e57c82/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T12:15:55,788 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened c910d27fbf3265165c7fd59609e57c82; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64122352, jitterRate=-0.04450249671936035}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:15:55,788 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c910d27fbf3265165c7fd59609e57c82 2024-12-09T12:15:55,788 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for c910d27fbf3265165c7fd59609e57c82: Running coprocessor pre-open hook at 1733746555746Writing region info on filesystem at 1733746555746Initializing all the Stores at 1733746555749 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746555750 (+1 ms)Cleaning up temporary data from old regions at 1733746555777 (+27 ms)Running coprocessor post-open hooks at 1733746555788 (+11 ms)Region opened successfully at 1733746555788 2024-12-09T12:15:55,790 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=e5251eee3971dde9f0f276cfd8632c21, regionState=OPEN, openSeqNum=2, regionLocation=f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:55,790 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1733746552866.c910d27fbf3265165c7fd59609e57c82., pid=36, masterSystemTime=1733746555291 2024-12-09T12:15:55,790 INFO [StoreOpener-62a60a0dff0c026a110e755e6c249001-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 62a60a0dff0c026a110e755e6c249001 2024-12-09T12:15:55,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure e5251eee3971dde9f0f276cfd8632c21, server=f15dd41c88ad,33783,1733746548634 because future has completed 2024-12-09T12:15:55,801 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1733746552866.c910d27fbf3265165c7fd59609e57c82. 2024-12-09T12:15:55,801 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,ggg,1733746552866.c910d27fbf3265165c7fd59609e57c82. 2024-12-09T12:15:55,802 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f15dd41c88ad,38111,1733746548467, table=test, region=d9e23742363329565223433d05d86296. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-09T12:15:55,803 INFO [StoreOpener-62a60a0dff0c026a110e755e6c249001-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62a60a0dff0c026a110e755e6c249001 columnFamilyName family 2024-12-09T12:15:55,803 DEBUG [StoreOpener-62a60a0dff0c026a110e755e6c249001-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:55,811 INFO [StoreOpener-62a60a0dff0c026a110e755e6c249001-1 {}] regionserver.HStore(327): Store=62a60a0dff0c026a110e755e6c249001/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:55,814 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 62a60a0dff0c026a110e755e6c249001 2024-12-09T12:15:55,818 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/62a60a0dff0c026a110e755e6c249001 2024-12-09T12:15:55,822 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/62a60a0dff0c026a110e755e6c249001 2024-12-09T12:15:55,826 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 62a60a0dff0c026a110e755e6c249001 2024-12-09T12:15:55,826 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 62a60a0dff0c026a110e755e6c249001 2024-12-09T12:15:55,832 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 62a60a0dff0c026a110e755e6c249001 2024-12-09T12:15:55,840 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/62a60a0dff0c026a110e755e6c249001/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T12:15:55,841 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 62a60a0dff0c026a110e755e6c249001; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60417443, jitterRate=-0.09970994293689728}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:15:55,841 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 62a60a0dff0c026a110e755e6c249001 2024-12-09T12:15:55,841 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 62a60a0dff0c026a110e755e6c249001: Running coprocessor pre-open hook at 1733746555785Writing region info on filesystem at 1733746555785Initializing all the Stores at 1733746555787 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746555787Cleaning up temporary data from old regions at 1733746555827 (+40 ms)Running coprocessor post-open hooks at 1733746555841 (+14 ms)Region opened successfully at 1733746555841 2024-12-09T12:15:55,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=9 2024-12-09T12:15:55,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure e5251eee3971dde9f0f276cfd8632c21, server=f15dd41c88ad,33783,1733746548634 in 362 msec 2024-12-09T12:15:55,844 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1733746552866.62a60a0dff0c026a110e755e6c249001., pid=49, masterSystemTime=1733746555465 2024-12-09T12:15:55,855 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1733746552866.62a60a0dff0c026a110e755e6c249001. 2024-12-09T12:15:55,855 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,mmm,1733746552866.62a60a0dff0c026a110e755e6c249001. 2024-12-09T12:15:55,855 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,ppp,1733746552866.43f38f304815b66d00adc6292e217f1a. 2024-12-09T12:15:55,855 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f15dd41c88ad,33783,1733746548634, table=test, region=ebee1500f62faadfac8720740a1e527e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-09T12:15:55,855 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 43f38f304815b66d00adc6292e217f1a, NAME => 'test,ppp,1733746552866.43f38f304815b66d00adc6292e217f1a.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-12-09T12:15:55,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, ASSIGN in 984 msec 2024-12-09T12:15:55,856 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 43f38f304815b66d00adc6292e217f1a 2024-12-09T12:15:55,856 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,ppp,1733746552866.43f38f304815b66d00adc6292e217f1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:55,856 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 43f38f304815b66d00adc6292e217f1a 2024-12-09T12:15:55,856 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f15dd41c88ad,36579,1733746548727, table=test, region=9ac6650f2a35b2c131e93ead260da106. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-09T12:15:55,856 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 43f38f304815b66d00adc6292e217f1a 2024-12-09T12:15:55,863 INFO [StoreOpener-43f38f304815b66d00adc6292e217f1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 43f38f304815b66d00adc6292e217f1a 2024-12-09T12:15:55,864 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=62a60a0dff0c026a110e755e6c249001, regionState=OPEN, openSeqNum=2, regionLocation=f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:55,865 INFO [StoreOpener-43f38f304815b66d00adc6292e217f1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43f38f304815b66d00adc6292e217f1a columnFamilyName family 2024-12-09T12:15:55,865 DEBUG [StoreOpener-43f38f304815b66d00adc6292e217f1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:55,866 INFO [StoreOpener-43f38f304815b66d00adc6292e217f1a-1 {}] regionserver.HStore(327): Store=43f38f304815b66d00adc6292e217f1a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:55,867 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 43f38f304815b66d00adc6292e217f1a 2024-12-09T12:15:55,868 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/43f38f304815b66d00adc6292e217f1a 2024-12-09T12:15:55,868 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,33783,1733746548634 because future has completed 2024-12-09T12:15:55,869 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/43f38f304815b66d00adc6292e217f1a 2024-12-09T12:15:55,872 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 43f38f304815b66d00adc6292e217f1a 2024-12-09T12:15:55,873 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 43f38f304815b66d00adc6292e217f1a 2024-12-09T12:15:55,884 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 43f38f304815b66d00adc6292e217f1a 2024-12-09T12:15:55,890 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/43f38f304815b66d00adc6292e217f1a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T12:15:55,892 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 43f38f304815b66d00adc6292e217f1a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61557981, jitterRate=-0.08271460235118866}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:15:55,892 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 43f38f304815b66d00adc6292e217f1a 2024-12-09T12:15:55,892 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 43f38f304815b66d00adc6292e217f1a: Running coprocessor pre-open hook at 1733746555856Writing region info on filesystem at 1733746555856Initializing all the Stores at 1733746555860 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746555861 (+1 ms)Cleaning up temporary data from old regions at 1733746555873 (+12 ms)Running coprocessor post-open hooks at 1733746555892 (+19 ms)Region opened successfully at 1733746555892 2024-12-09T12:15:55,895 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1733746552866.43f38f304815b66d00adc6292e217f1a., pid=52, masterSystemTime=1733746555465 2024-12-09T12:15:55,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=17 2024-12-09T12:15:55,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,33783,1733746548634 in 555 msec 2024-12-09T12:15:55,909 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1733746552866.43f38f304815b66d00adc6292e217f1a. 2024-12-09T12:15:55,909 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,ppp,1733746552866.43f38f304815b66d00adc6292e217f1a. 2024-12-09T12:15:55,913 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f15dd41c88ad,38111,1733746548467, table=test, region=d9e23742363329565223433d05d86296. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-09T12:15:55,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN in 1.0500 sec 2024-12-09T12:15:55,938 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=43f38f304815b66d00adc6292e217f1a, regionState=OPEN, openSeqNum=2, regionLocation=f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:55,944 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=c910d27fbf3265165c7fd59609e57c82, regionState=OPEN, openSeqNum=2, regionLocation=f15dd41c88ad,36579,1733746548727 2024-12-09T12:15:55,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43f38f304815b66d00adc6292e217f1a, server=f15dd41c88ad,33783,1733746548634 because future has completed 2024-12-09T12:15:55,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure c910d27fbf3265165c7fd59609e57c82, server=f15dd41c88ad,36579,1733746548727 because future has completed 2024-12-09T12:15:55,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=6 2024-12-09T12:15:55,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 9ac6650f2a35b2c131e93ead260da106, server=f15dd41c88ad,36579,1733746548727 in 811 msec 2024-12-09T12:15:55,956 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9ac6650f2a35b2c131e93ead260da106, ASSIGN in 1.0940 sec 2024-12-09T12:15:55,959 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f15dd41c88ad,33783,1733746548634, table=test, region=ebee1500f62faadfac8720740a1e527e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-09T12:15:55,959 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f15dd41c88ad,33783,1733746548634, table=test, region=43f38f304815b66d00adc6292e217f1a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-09T12:15:55,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=11 2024-12-09T12:15:55,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure c910d27fbf3265165c7fd59609e57c82, server=f15dd41c88ad,36579,1733746548727 in 814 msec 2024-12-09T12:15:55,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c910d27fbf3265165c7fd59609e57c82, ASSIGN in 1.1050 sec 2024-12-09T12:15:55,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=20 2024-12-09T12:15:55,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 43f38f304815b66d00adc6292e217f1a, server=f15dd41c88ad,33783,1733746548634 in 615 msec 2024-12-09T12:15:55,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN in 1.1160 sec 2024-12-09T12:15:55,979 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=23280f1743db631eaa35bf3487e69147, regionState=OPEN, openSeqNum=2, regionLocation=f15dd41c88ad,38111,1733746548467 2024-12-09T12:15:55,983 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23280f1743db631eaa35bf3487e69147, server=f15dd41c88ad,38111,1733746548467 because future has completed 2024-12-09T12:15:55,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=14 2024-12-09T12:15:55,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure d9e23742363329565223433d05d86296, server=f15dd41c88ad,38111,1733746548467 in 757 msec 2024-12-09T12:15:55,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d9e23742363329565223433d05d86296, ASSIGN in 1.1280 sec 2024-12-09T12:15:55,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=26 2024-12-09T12:15:55,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 23280f1743db631eaa35bf3487e69147, server=f15dd41c88ad,38111,1733746548467 in 736 msec 2024-12-09T12:15:55,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23280f1743db631eaa35bf3487e69147, ASSIGN in 1.1370 sec 2024-12-09T12:15:56,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=16 2024-12-09T12:15:56,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure ebee1500f62faadfac8720740a1e527e, server=f15dd41c88ad,33783,1733746548634 in 649 msec 2024-12-09T12:15:56,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN in 1.1440 sec 2024-12-09T12:15:56,006 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=a258d2ecd9a494e8d56406b774da28ef, regionState=OPEN, openSeqNum=2, regionLocation=f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:56,011 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=6b5306e854c5aec62eff62a18cec5859, regionState=OPEN, openSeqNum=2, regionLocation=f15dd41c88ad,38111,1733746548467 2024-12-09T12:15:56,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure a258d2ecd9a494e8d56406b774da28ef, server=f15dd41c88ad,33783,1733746548634 because future has completed 2024-12-09T12:15:56,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6b5306e854c5aec62eff62a18cec5859, server=f15dd41c88ad,38111,1733746548467 because future has completed 2024-12-09T12:15:56,019 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f15dd41c88ad,38111,1733746548467, table=test, region=6b5306e854c5aec62eff62a18cec5859. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-09T12:15:56,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=12 2024-12-09T12:15:56,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure a258d2ecd9a494e8d56406b774da28ef, server=f15dd41c88ad,33783,1733746548634 in 681 msec 2024-12-09T12:15:56,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN in 1.1610 sec 2024-12-09T12:15:56,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=10 2024-12-09T12:15:56,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 6b5306e854c5aec62eff62a18cec5859, server=f15dd41c88ad,38111,1733746548467 in 830 msec 2024-12-09T12:15:56,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6b5306e854c5aec62eff62a18cec5859, ASSIGN in 1.1820 sec 2024-12-09T12:15:56,045 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=8ceebab6d8b44ccd74ad65e871405953, regionState=OPEN, openSeqNum=2, regionLocation=f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:56,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ceebab6d8b44ccd74ad65e871405953, server=f15dd41c88ad,33783,1733746548634 because future has completed 2024-12-09T12:15:56,057 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1ce28e8d0ce113d2637ab328f5ffd3be, regionState=OPEN, openSeqNum=2, regionLocation=f15dd41c88ad,36579,1733746548727 2024-12-09T12:15:56,061 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ce28e8d0ce113d2637ab328f5ffd3be, server=f15dd41c88ad,36579,1733746548727 because future has completed 2024-12-09T12:15:56,063 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f15dd41c88ad,33783,1733746548634, table=test, region=8ceebab6d8b44ccd74ad65e871405953. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-09T12:15:56,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=8 2024-12-09T12:15:56,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1ce28e8d0ce113d2637ab328f5ffd3be, server=f15dd41c88ad,36579,1733746548727 in 937 msec 2024-12-09T12:15:56,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1ce28e8d0ce113d2637ab328f5ffd3be, ASSIGN in 1.2100 sec 2024-12-09T12:15:56,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=27 2024-12-09T12:15:56,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 8ceebab6d8b44ccd74ad65e871405953, server=f15dd41c88ad,33783,1733746548634 in 778 msec 2024-12-09T12:15:56,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN in 1.2300 sec 2024-12-09T12:15:56,093 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8bc29ffded7f1b74a1ee2986c5e611f8, regionState=OPENING, regionLocation=f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:56,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN because future has completed 2024-12-09T12:15:56,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8bc29ffded7f1b74a1ee2986c5e611f8, server=f15dd41c88ad,33783,1733746548634}] 2024-12-09T12:15:56,260 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,ccc,1733746552866.8bc29ffded7f1b74a1ee2986c5e611f8. 2024-12-09T12:15:56,260 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 8bc29ffded7f1b74a1ee2986c5e611f8, NAME => 'test,ccc,1733746552866.8bc29ffded7f1b74a1ee2986c5e611f8.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-12-09T12:15:56,261 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8bc29ffded7f1b74a1ee2986c5e611f8 2024-12-09T12:15:56,261 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,ccc,1733746552866.8bc29ffded7f1b74a1ee2986c5e611f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:56,261 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 8bc29ffded7f1b74a1ee2986c5e611f8 2024-12-09T12:15:56,261 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 8bc29ffded7f1b74a1ee2986c5e611f8 2024-12-09T12:15:56,270 INFO [StoreOpener-8bc29ffded7f1b74a1ee2986c5e611f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8bc29ffded7f1b74a1ee2986c5e611f8 2024-12-09T12:15:56,272 INFO [StoreOpener-8bc29ffded7f1b74a1ee2986c5e611f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8bc29ffded7f1b74a1ee2986c5e611f8 columnFamilyName family 2024-12-09T12:15:56,272 DEBUG [StoreOpener-8bc29ffded7f1b74a1ee2986c5e611f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:56,273 INFO [StoreOpener-8bc29ffded7f1b74a1ee2986c5e611f8-1 {}] regionserver.HStore(327): Store=8bc29ffded7f1b74a1ee2986c5e611f8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:56,273 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 8bc29ffded7f1b74a1ee2986c5e611f8 2024-12-09T12:15:56,274 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/8bc29ffded7f1b74a1ee2986c5e611f8 2024-12-09T12:15:56,275 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/8bc29ffded7f1b74a1ee2986c5e611f8 2024-12-09T12:15:56,275 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 8bc29ffded7f1b74a1ee2986c5e611f8 2024-12-09T12:15:56,275 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 8bc29ffded7f1b74a1ee2986c5e611f8 2024-12-09T12:15:56,278 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 8bc29ffded7f1b74a1ee2986c5e611f8 2024-12-09T12:15:56,289 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/8bc29ffded7f1b74a1ee2986c5e611f8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T12:15:56,290 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 8bc29ffded7f1b74a1ee2986c5e611f8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71281422, jitterRate=0.06217595934867859}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:15:56,290 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8bc29ffded7f1b74a1ee2986c5e611f8 2024-12-09T12:15:56,290 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 8bc29ffded7f1b74a1ee2986c5e611f8: Running coprocessor pre-open hook at 1733746556261Writing region info on filesystem at 1733746556261Initializing all the Stores at 1733746556267 (+6 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746556267Cleaning up temporary data from old regions at 1733746556275 (+8 ms)Running coprocessor post-open hooks at 1733746556290 (+15 ms)Region opened successfully at 1733746556290 2024-12-09T12:15:56,298 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1733746552866.8bc29ffded7f1b74a1ee2986c5e611f8., pid=56, masterSystemTime=1733746556253 2024-12-09T12:15:56,303 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1733746552866.8bc29ffded7f1b74a1ee2986c5e611f8. 2024-12-09T12:15:56,303 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,ccc,1733746552866.8bc29ffded7f1b74a1ee2986c5e611f8. 2024-12-09T12:15:56,306 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8bc29ffded7f1b74a1ee2986c5e611f8, regionState=OPEN, openSeqNum=2, regionLocation=f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:56,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8bc29ffded7f1b74a1ee2986c5e611f8, server=f15dd41c88ad,33783,1733746548634 because future has completed 2024-12-09T12:15:56,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=7 2024-12-09T12:15:56,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 8bc29ffded7f1b74a1ee2986c5e611f8, server=f15dd41c88ad,33783,1733746548634 in 227 msec 2024-12-09T12:15:56,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=7, resume processing ppid=4 2024-12-09T12:15:56,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN in 1.4730 sec 2024-12-09T12:15:56,339 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T12:15:56,340 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733746556339"}]},"ts":"1733746556339"} 2024-12-09T12:15:56,354 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-12-09T12:15:56,363 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T12:15:56,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 3.4750 sec 2024-12-09T12:15:56,795 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-12-09T12:15:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T12:15:57,153 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-12-09T12:15:57,153 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-12-09T12:15:57,155 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T12:15:57,175 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-12-09T12:15:57,176 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T12:15:57,177 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-12-09T12:15:57,178 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T12:15:57,183 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-12-09T12:15:57,204 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-12-09T12:15:57,222 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T12:15:57,226 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T12:15:57,229 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T12:15:57,231 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T12:15:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8bc29ffded7f1b74a1ee2986c5e611f8 changed from -1.0 to 0.0, refreshing cache 2024-12-09T12:15:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e5251eee3971dde9f0f276cfd8632c21 changed from -1.0 to 0.0, refreshing cache 2024-12-09T12:15:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a258d2ecd9a494e8d56406b774da28ef changed from -1.0 to 0.0, refreshing cache 2024-12-09T12:15:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ebee1500f62faadfac8720740a1e527e changed from -1.0 to 0.0, refreshing cache 2024-12-09T12:15:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 62a60a0dff0c026a110e755e6c249001 changed from -1.0 to 0.0, refreshing cache 2024-12-09T12:15:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 43f38f304815b66d00adc6292e217f1a changed from -1.0 to 0.0, refreshing cache 2024-12-09T12:15:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ba4a26e129719340e5696ac56c6b61ac changed from -1.0 to 0.0, refreshing cache 2024-12-09T12:15:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8ceebab6d8b44ccd74ad65e871405953 changed from -1.0 to 0.0, refreshing cache 2024-12-09T12:15:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c8f8bc12489e5c0370ba59c751783a23 changed from -1.0 to 0.0, refreshing cache 2024-12-09T12:15:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9ac6650f2a35b2c131e93ead260da106 changed from -1.0 to 0.0, refreshing cache 2024-12-09T12:15:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1ce28e8d0ce113d2637ab328f5ffd3be changed from -1.0 to 0.0, refreshing cache 2024-12-09T12:15:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c910d27fbf3265165c7fd59609e57c82 changed from -1.0 to 0.0, refreshing cache 2024-12-09T12:15:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c9702c569de70ef36018a26943c3226f changed from -1.0 to 0.0, refreshing cache 2024-12-09T12:15:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fdcb3d892a0c9cf641eaedbbd9ec5937 changed from -1.0 to 0.0, refreshing cache 2024-12-09T12:15:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fab7cda743d5cc81945ffb7c431d6346 changed from -1.0 to 0.0, refreshing cache 2024-12-09T12:15:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 603cf0cbb0e068a5f7727e4abab3a5d5 changed from -1.0 to 0.0, refreshing cache 2024-12-09T12:15:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b0351c7e96a16dae18ff5a8d1664b2b1 changed from -1.0 to 0.0, refreshing cache 2024-12-09T12:15:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6a763ac99f0e3a37f94a31b5f0479530 changed from -1.0 to 0.0, refreshing cache 2024-12-09T12:15:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region cc7aaad28378d11daf90725c0f9d39f5 changed from -1.0 to 0.0, refreshing cache 2024-12-09T12:15:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6b5306e854c5aec62eff62a18cec5859 changed from -1.0 to 0.0, refreshing cache 2024-12-09T12:15:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a0cf25c9917186a9e3dbd87e44f33615 changed from -1.0 to 0.0, refreshing cache 2024-12-09T12:15:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d9e23742363329565223433d05d86296 changed from -1.0 to 0.0, refreshing cache 2024-12-09T12:15:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3a1205a417c23329b514f783f4727022 changed from -1.0 to 0.0, refreshing cache 2024-12-09T12:15:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e0adbf1ba01f94cad79f2814d407182f changed from -1.0 to 0.0, refreshing cache 2024-12-09T12:15:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 23280f1743db631eaa35bf3487e69147 changed from -1.0 to 0.0, refreshing cache 2024-12-09T12:15:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7eaca6c428d8bff0cd952b3c2dfe5408 changed from -1.0 to 0.0, refreshing cache 2024-12-09T12:15:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-12-09T12:15:57,313 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-12-09T12:15:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-12-09T12:15:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.BalancerClusterState(204): Hosts are {f15dd41c88ad=0} racks are {/default-rack=0} 2024-12-09T12:15:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.BalancerClusterState(303): server 0 has 10 regions 2024-12-09T12:15:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-12-09T12:15:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.BalancerClusterState(303): server 2 has 8 regions 2024-12-09T12:15:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T12:15:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T12:15:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-09T12:15:57,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T12:15:57,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T12:15:57,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-09T12:15:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-12-09T12:15:57,319 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.StochasticLoadBalancer(403): Cluster wide - Calculating plan. may take up to 30000ms to complete. 2024-12-09T12:15:57,320 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.StochasticLoadBalancer(515): Start StochasticLoadBalancer.balancer, initial weighted average imbalance=0.06931633435746164, functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.06415002990995843, need balance); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); computedMaxSteps=64800 2024-12-09T12:15:57,370 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T12:15:57,457 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-09T12:15:57,634 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-09T12:15:57,635 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-09T12:15:57,637 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test 2024-12-09T12:15:57,637 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test Metrics about Tables on a single HBase RegionServer 2024-12-09T12:15:57,638 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T12:15:57,638 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-09T12:15:57,638 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-09T12:15:57,638 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-09T12:15:57,987 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] balancer.StochasticLoadBalancer(562): Finished computing new moving plan. Computation took 671 ms to try 64800 different iterations. Found a solution that moves 1 regions; Going from a computed imbalance of 0.06931633435746164 to a new imbalance of 0.01690158032826896. funtionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.037037037037037035, need balance); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-12-09T12:15:57,992 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] master.HMaster(2167): Balancer plans size is 1, the balance interval is 300000 ms, and the max number regions in transition is 27 2024-12-09T12:15:57,992 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] master.HMaster(2172): balance hri=e5251eee3971dde9f0f276cfd8632c21, source=f15dd41c88ad,33783,1733746548634, destination=f15dd41c88ad,38111,1733746548467 2024-12-09T12:15:58,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, REOPEN/MOVE 2024-12-09T12:15:58,002 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, REOPEN/MOVE 2024-12-09T12:15:58,015 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=e5251eee3971dde9f0f276cfd8632c21, regionState=CLOSING, regionLocation=f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:58,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, REOPEN/MOVE because future has completed 2024-12-09T12:15:58,031 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T12:15:58,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure e5251eee3971dde9f0f276cfd8632c21, server=f15dd41c88ad,33783,1733746548634}] 2024-12-09T12:15:58,199 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(122): Close e5251eee3971dde9f0f276cfd8632c21 2024-12-09T12:15:58,199 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T12:15:58,200 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1722): Closing e5251eee3971dde9f0f276cfd8632c21, disabling compactions & flushes 2024-12-09T12:15:58,200 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1755): Closing region test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21. 2024-12-09T12:15:58,200 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21. 2024-12-09T12:15:58,200 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1843): Acquired close lock on test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21. after waiting 0 ms 2024-12-09T12:15:58,200 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1853): Updates disabled for region test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21. 2024-12-09T12:15:58,243 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/e5251eee3971dde9f0f276cfd8632c21/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T12:15:58,247 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1973): Closed test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21. 2024-12-09T12:15:58,247 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1676): Region close journal for e5251eee3971dde9f0f276cfd8632c21: Waiting for close lock at 1733746558200Running coprocessor pre-close hooks at 1733746558200Disabling compacts and flushes for region at 1733746558200Disabling writes for close at 1733746558200Writing region close event to WAL at 1733746558207 (+7 ms)Running coprocessor post-close hooks at 1733746558244 (+37 ms)Closed at 1733746558247 (+3 ms) 2024-12-09T12:15:58,248 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionServer(3302): Adding e5251eee3971dde9f0f276cfd8632c21 move to f15dd41c88ad,38111,1733746548467 record at close sequenceid=2 2024-12-09T12:15:58,258 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(157): Closed e5251eee3971dde9f0f276cfd8632c21 2024-12-09T12:15:58,259 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=e5251eee3971dde9f0f276cfd8632c21, regionState=CLOSED 2024-12-09T12:15:58,271 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure e5251eee3971dde9f0f276cfd8632c21, server=f15dd41c88ad,33783,1733746548634 because future has completed 2024-12-09T12:15:58,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-12-09T12:15:58,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; CloseRegionProcedure e5251eee3971dde9f0f276cfd8632c21, server=f15dd41c88ad,33783,1733746548634 in 243 msec 2024-12-09T12:15:58,287 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, REOPEN/MOVE; state=CLOSED, location=f15dd41c88ad,38111,1733746548467; forceNewPlan=false, retain=false 2024-12-09T12:15:58,438 INFO [f15dd41c88ad:42519 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T12:15:58,438 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=e5251eee3971dde9f0f276cfd8632c21, regionState=OPENING, regionLocation=f15dd41c88ad,38111,1733746548467 2024-12-09T12:15:58,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, REOPEN/MOVE because future has completed 2024-12-09T12:15:58,442 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure e5251eee3971dde9f0f276cfd8632c21, server=f15dd41c88ad,38111,1733746548467}] 2024-12-09T12:15:58,609 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21. 2024-12-09T12:15:58,609 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => e5251eee3971dde9f0f276cfd8632c21, NAME => 'test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-12-09T12:15:58,610 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e5251eee3971dde9f0f276cfd8632c21 2024-12-09T12:15:58,610 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:58,610 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for e5251eee3971dde9f0f276cfd8632c21 2024-12-09T12:15:58,610 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for e5251eee3971dde9f0f276cfd8632c21 2024-12-09T12:15:58,615 INFO [StoreOpener-e5251eee3971dde9f0f276cfd8632c21-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e5251eee3971dde9f0f276cfd8632c21 2024-12-09T12:15:58,617 INFO [StoreOpener-e5251eee3971dde9f0f276cfd8632c21-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5251eee3971dde9f0f276cfd8632c21 columnFamilyName family 2024-12-09T12:15:58,617 DEBUG [StoreOpener-e5251eee3971dde9f0f276cfd8632c21-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:58,619 INFO [StoreOpener-e5251eee3971dde9f0f276cfd8632c21-1 {}] regionserver.HStore(327): Store=e5251eee3971dde9f0f276cfd8632c21/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:15:58,619 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for e5251eee3971dde9f0f276cfd8632c21 2024-12-09T12:15:58,620 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/e5251eee3971dde9f0f276cfd8632c21 2024-12-09T12:15:58,624 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/e5251eee3971dde9f0f276cfd8632c21 2024-12-09T12:15:58,625 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for e5251eee3971dde9f0f276cfd8632c21 2024-12-09T12:15:58,625 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for e5251eee3971dde9f0f276cfd8632c21 2024-12-09T12:15:58,630 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for e5251eee3971dde9f0f276cfd8632c21 2024-12-09T12:15:58,632 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened e5251eee3971dde9f0f276cfd8632c21; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67947183, jitterRate=0.012491926550865173}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:15:58,632 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e5251eee3971dde9f0f276cfd8632c21 2024-12-09T12:15:58,632 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for e5251eee3971dde9f0f276cfd8632c21: Running coprocessor pre-open hook at 1733746558610Writing region info on filesystem at 1733746558610Initializing all the Stores at 1733746558614 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746558614Cleaning up temporary data from old regions at 1733746558625 (+11 ms)Running coprocessor post-open hooks at 1733746558632 (+7 ms)Region opened successfully at 1733746558632 2024-12-09T12:15:58,635 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21., pid=59, masterSystemTime=1733746558595 2024-12-09T12:15:58,645 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21. 2024-12-09T12:15:58,645 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21. 2024-12-09T12:15:58,647 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=e5251eee3971dde9f0f276cfd8632c21, regionState=OPEN, openSeqNum=5, regionLocation=f15dd41c88ad,38111,1733746548467 2024-12-09T12:15:58,664 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure e5251eee3971dde9f0f276cfd8632c21, server=f15dd41c88ad,38111,1733746548467 because future has completed 2024-12-09T12:15:58,670 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=57 2024-12-09T12:15:58,670 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=57, state=SUCCESS, hasLock=false; OpenRegionProcedure e5251eee3971dde9f0f276cfd8632c21, server=f15dd41c88ad,38111,1733746548467 in 224 msec 2024-12-09T12:15:58,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, REOPEN/MOVE in 677 msec 2024-12-09T12:15:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-12-09T12:15:58,707 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T12:15:58,718 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.3 set balanceSwitch=false 2024-12-09T12:15:58,747 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=380, OpenFileDescriptor=645, MaxFileDescriptor=1048576, SystemLoadAverage=898, ProcessCount=11, AvailableMemoryMB=5183 2024-12-09T12:15:58,751 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-12-09T12:15:58,751 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-12-09T12:15:58,752 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:58,752 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server f15dd41c88ad,33783,1733746548634: Simulated kill ***** 2024-12-09T12:15:58,753 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-09T12:15:58,755 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-09T12:15:58,757 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-09T12:15:58,759 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-09T12:15:58,760 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-09T12:15:58,768 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 354275784 }, "NonHeapMemoryUsage": { "committed": 154337280, "init": 7667712, "max": -1, "used": 149674960 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "f15dd41c88ad", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 2, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 34182, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 98, "ProcessCallTime_min": 0, "ProcessCallTime_max": 35, "ProcessCallTime_mean": 5, "ProcessCallTime_25th_percentile": 8, "ProcessCallTime_median": 17, "ProcessCallTime_75th_percentile": 26, "ProcessCallTime_90th_percentile": 31, "ProcessCallTime_95th_percentile": 33, "ProcessCallTime_98th_percentile": 34, "ProcessCallTime_99th_percentile": 34, "ProcessCallTime_99.9th_percentile": 34, "ProcessCallTime_TimeRangeCount_0-1": 98, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 98, "QueueCallTime_min": 0, "QueueCallTime_max": 76, "QueueCallTime_mean": 10, "QueueCallTime_25th_percentile": 19, "QueueCallTime_median": 38, "QueueCallTime_75th_percentile": 57, "QueueCallTime_90th_percentile": 68, "QueueCallTime_95th_percentile": 72, "QueueCallTime_98th_percentile": 74, "QueueCallTime_99th_percentile": 75, "QueueCallTime_99.9th_percentile": 75, "QueueCallTime_TimeRangeCount_0-1": 98, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 98, "TotalCallTime_min": 0, "TotalCallTime_max": 80, "TotalCallTime_mean": 15, "TotalCallTime_25th_percentile": 20, "TotalCallTime_median": 40, "TotalCallTime_75th_percentile": 60, "TotalCallTime_90th_percentile": 72, "TotalCallTime_95th_percentile": 76, "TotalCallTime_98th_percentile": 78, "TotalCallTime_99th_percentile": 79, "TotalCallTime_99.9th_percentile": 79, "TotalCallTime_TimeRangeCount_0-1": 98, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 98, "ResponseSize_min": 0, "ResponseSize_max": 22358, "ResponseSize_mean": 697, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16768, "ResponseSize_90th_percentile": 20122, "ResponseSize_95th_percentile": 21240, "ResponseSize_98th_percentile": 21910, "ResponseSize_99th_percentile": 22134, "ResponseSize_99.9th_percentile": 22335, "ResponseSize_SizeRangeCount_0-10": 98, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 98, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 316, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 98, "sentBytes": 54700 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "f15dd41c88ad", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:60469", "tag.serverName": "f15dd41c88ad,38111,1733746548467", "tag.clusterId": "bd14337c-7e24-4686-a8c1-6c16b6d09ea0", "tag.Context": "regionserver", "tag.Hostname": "f15dd41c88ad", "regionCount": 8, "storeCount": 8, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2048, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733746548467, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 0, "localBytesRead": 0, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 0, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 8, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 8, "rpcScanRequestCount": 8, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 8, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 8, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 8, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 8, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-09T12:15:58,773 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42519 {}] master.MasterRpcServices(700): f15dd41c88ad,33783,1733746548634 reported a fatal error: ***** ABORTING region server f15dd41c88ad,33783,1733746548634: Simulated kill ***** 2024-12-09T12:15:58,779 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f15dd41c88ad,33783,1733746548634' ***** 2024-12-09T12:15:58,780 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-12-09T12:15:58,780 INFO [RS:1;f15dd41c88ad:33783 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T12:15:58,780 INFO [RS:1;f15dd41c88ad:33783 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-09T12:15:58,780 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T12:15:58,780 INFO [RS:1;f15dd41c88ad:33783 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-09T12:15:58,781 INFO [RS:1;f15dd41c88ad:33783 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T12:15:58,781 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T12:15:58,781 INFO [RS:1;f15dd41c88ad:33783 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;f15dd41c88ad:33783. 2024-12-09T12:15:58,781 DEBUG [RS:1;f15dd41c88ad:33783 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T12:15:58,788 DEBUG [RS:1;f15dd41c88ad:33783 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T12:15:58,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741836_1012 (size=33749) 2024-12-09T12:15:58,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741836_1012 (size=33749) 2024-12-09T12:15:58,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741836_1012 (size=33749) 2024-12-09T12:15:58,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741834_1010 (size=2774) 2024-12-09T12:15:58,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741834_1010 (size=2774) 2024-12-09T12:15:58,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741834_1010 (size=2774) 2024-12-09T12:15:58,806 DEBUG [RS:1;f15dd41c88ad:33783 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T12:15:58,806 INFO [RS:1;f15dd41c88ad:33783 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T12:15:58,807 INFO [RS:1;f15dd41c88ad:33783 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:33783 2024-12-09T12:15:58,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T12:15:58,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33783-0x1012b1c64ac0002, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f15dd41c88ad,33783,1733746548634 2024-12-09T12:15:58,822 INFO [RS:1;f15dd41c88ad:33783 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T12:15:58,823 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f15dd41c88ad,33783,1733746548634] 2024-12-09T12:15:58,829 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f15dd41c88ad,33783,1733746548634 already deleted, retry=false 2024-12-09T12:15:58,829 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of f15dd41c88ad,33783,1733746548634 on f15dd41c88ad,42519,1733746546940 2024-12-09T12:15:58,833 INFO [regionserver/f15dd41c88ad:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T12:15:58,839 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true 2024-12-09T12:15:58,843 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=60 for f15dd41c88ad,33783,1733746548634 (carryingMeta=true) f15dd41c88ad,33783,1733746548634/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@203dd489[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-09T12:15:58,844 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true 2024-12-09T12:15:58,848 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true, isMeta: true 2024-12-09T12:15:58,852 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634-splitting 2024-12-09T12:15:58,854 INFO [PEWorker-1 {}] master.SplitWALManager(105): f15dd41c88ad,33783,1733746548634 WAL count=1, meta=true 2024-12-09T12:15:58,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta}] 2024-12-09T12:15:58,864 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:15:58,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727}] 2024-12-09T12:15:58,882 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T12:15:58,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33783-0x1012b1c64ac0002, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T12:15:58,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33783-0x1012b1c64ac0002, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T12:15:58,932 INFO [RS:1;f15dd41c88ad:33783 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T12:15:58,932 INFO [RS:1;f15dd41c88ad:33783 {}] regionserver.HRegionServer(1031): Exiting; stopping=f15dd41c88ad,33783,1733746548634; zookeeper connection closed. 2024-12-09T12:15:58,932 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@e4bf8f6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@e4bf8f6 2024-12-09T12:15:59,002 INFO [regionserver/f15dd41c88ad:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-09T12:15:59,002 INFO [regionserver/f15dd41c88ad:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-09T12:15:59,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36579 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=62 2024-12-09T12:15:59,063 INFO [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(299): Splitting hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634-splitting/f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, size=33.0 K (33749bytes) 2024-12-09T12:15:59,064 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-09T12:15:59,064 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-09T12:15:59,065 INFO [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634-splitting/f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta 2024-12-09T12:15:59,070 INFO [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634-splitting/f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta after 3ms 2024-12-09T12:15:59,079 INFO [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(310): Open hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634-splitting/f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta took 17ms 2024-12-09T12:15:59,090 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-12-09T12:15:59,097 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634-splitting/f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta so closing down 2024-12-09T12:15:59,098 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T12:15:59,100 INFO [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta.temp 2024-12-09T12:15:59,103 INFO [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta.temp 2024-12-09T12:15:59,106 INFO [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T12:15:59,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741864_1040 (size=33508) 2024-12-09T12:15:59,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741864_1040 (size=33508) 2024-12-09T12:15:59,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741864_1040 (size=33508) 2024-12-09T12:15:59,118 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta.temp (wrote 61 edits, skipped 0 edits in 2 ms) 2024-12-09T12:15:59,120 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta.temp to hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-12-09T12:15:59,124 INFO [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(425): Processed 62 edits across 1 Regions in 39 ms; skipped=1; WAL=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634-splitting/f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false 2024-12-09T12:15:59,124 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(428): Completed split of hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634-splitting/f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, journal: Splitting hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634-splitting/f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, size=33.0 K (33749bytes) at 1733746559063Finishing writing output for hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634-splitting/f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta so closing down at 1733746559097 (+34 ms)Creating recovered edits writer path=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta.temp at 1733746559103 (+6 ms)3 split writer threads finished at 1733746559106 (+3 ms)Closed recovered edits writer path=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta.temp (wrote 61 edits, skipped 0 edits in 2 ms) at 1733746559118 (+12 ms)Rename recovered edits hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta.temp to hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 at 1733746559120 (+2 ms)Processed 62 edits across 1 Regions in 39 ms; skipped=1; WAL=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634-splitting/f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false at 1733746559124 (+4 ms) 2024-12-09T12:15:59,124 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634-splitting/f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta 2024-12-09T12:15:59,125 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-12-09T12:15:59,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-12-09T12:15:59,134 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634-splitting/f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta to hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/oldWALs 2024-12-09T12:15:59,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-12-09T12:15:59,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 in 267 msec 2024-12-09T12:15:59,138 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:15:59,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-09T12:15:59,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 in 283 msec 2024-12-09T12:15:59,144 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634-splitting dir is empty, no logs to split. 2024-12-09T12:15:59,144 INFO [PEWorker-1 {}] master.SplitWALManager(105): f15dd41c88ad,33783,1733746548634 WAL count=0, meta=true 2024-12-09T12:15:59,144 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if f15dd41c88ad,33783,1733746548634 WAL splitting is done? wals=0, meta=true 2024-12-09T12:15:59,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T12:15:59,147 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T12:15:59,148 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-09T12:15:59,298 DEBUG [f15dd41c88ad:42519 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-09T12:15:59,298 DEBUG [f15dd41c88ad:42519 {}] balancer.BalancerClusterState(204): Hosts are {f15dd41c88ad=0} racks are {/default-rack=0} 2024-12-09T12:15:59,299 DEBUG [f15dd41c88ad:42519 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T12:15:59,299 DEBUG [f15dd41c88ad:42519 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T12:15:59,299 DEBUG [f15dd41c88ad:42519 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T12:15:59,299 DEBUG [f15dd41c88ad:42519 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T12:15:59,299 INFO [f15dd41c88ad:42519 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T12:15:59,299 INFO [f15dd41c88ad:42519 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T12:15:59,299 DEBUG [f15dd41c88ad:42519 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T12:15:59,300 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f15dd41c88ad,38111,1733746548467 2024-12-09T12:15:59,301 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f15dd41c88ad,38111,1733746548467, state=OPENING 2024-12-09T12:15:59,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:15:59,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:15:59,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:15:59,303 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T12:15:59,304 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:15:59,304 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:15:59,304 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:15:59,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467}] 2024-12-09T12:15:59,463 INFO [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-09T12:15:59,464 INFO [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T12:15:59,464 INFO [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-09T12:15:59,466 INFO [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f15dd41c88ad%2C38111%2C1733746548467.meta, suffix=.meta, logDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,38111,1733746548467, archiveDir=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/oldWALs, maxLogs=32 2024-12-09T12:15:59,483 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,38111,1733746548467/f15dd41c88ad%2C38111%2C1733746548467.meta.1733746559468.meta, exclude list is [], retry=0 2024-12-09T12:15:59,487 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33961,DS-e1bc5ec1-d599-496b-8be4-5493e5661bf9,DISK] 2024-12-09T12:15:59,487 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39373,DS-0456a585-8dc2-4cb5-ae72-660bf02f0e80,DISK] 2024-12-09T12:15:59,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33963,DS-be4b45f7-c1d4-4431-b834-2678e63cb272,DISK] 2024-12-09T12:15:59,503 INFO [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,38111,1733746548467/f15dd41c88ad%2C38111%2C1733746548467.meta.1733746559468.meta 2024-12-09T12:15:59,510 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41475:41475),(127.0.0.1/127.0.0.1:42689:42689),(127.0.0.1/127.0.0.1:41543:41543)] 2024-12-09T12:15:59,511 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-09T12:15:59,511 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-09T12:15:59,511 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-09T12:15:59,512 INFO [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-09T12:15:59,512 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-09T12:15:59,512 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:15:59,512 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-09T12:15:59,512 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-09T12:15:59,517 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T12:15:59,519 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T12:15:59,519 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:59,520 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T12:15:59,520 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T12:15:59,521 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T12:15:59,521 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:59,522 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T12:15:59,522 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T12:15:59,523 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T12:15:59,523 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:59,524 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T12:15:59,524 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T12:15:59,525 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T12:15:59,525 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:15:59,526 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T12:15:59,526 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T12:15:59,537 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740 2024-12-09T12:15:59,540 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740 2024-12-09T12:15:59,541 INFO [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-12-09T12:15:59,565 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5793): Applied 277, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=64, path=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-12-09T12:15:59,569 INFO [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=29.41 KB heapSize=50.67 KB 2024-12-09T12:15:59,657 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/.tmp/info/9286a7f456714792a95bfc1814d3ac41 is 138, key is test,bbb,1733746552866.9ac6650f2a35b2c131e93ead260da106./info:regioninfo/1733746555754/Put/seqid=0 2024-12-09T12:15:59,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741866_1042 (size=36385) 2024-12-09T12:15:59,724 INFO [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.26 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/.tmp/info/9286a7f456714792a95bfc1814d3ac41 2024-12-09T12:15:59,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741866_1042 (size=36385) 2024-12-09T12:15:59,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741866_1042 (size=36385) 2024-12-09T12:15:59,809 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/.tmp/ns/3c2e522857e34724ab2b4fdcca44dbe8 is 43, key is default/ns:d/1733746552113/Put/seqid=0 2024-12-09T12:15:59,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741867_1043 (size=5153) 2024-12-09T12:15:59,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741867_1043 (size=5153) 2024-12-09T12:15:59,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741867_1043 (size=5153) 2024-12-09T12:15:59,826 INFO [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/.tmp/ns/3c2e522857e34724ab2b4fdcca44dbe8 2024-12-09T12:15:59,860 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/.tmp/table/1a1c62dd453048468bb3cb207f9dd13e is 40, key is test/table:state/1733746556339/Put/seqid=0 2024-12-09T12:15:59,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741868_1044 (size=5165) 2024-12-09T12:15:59,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741868_1044 (size=5165) 2024-12-09T12:15:59,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741868_1044 (size=5165) 2024-12-09T12:15:59,870 INFO [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/.tmp/table/1a1c62dd453048468bb3cb207f9dd13e 2024-12-09T12:15:59,882 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/.tmp/info/9286a7f456714792a95bfc1814d3ac41 as hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/info/9286a7f456714792a95bfc1814d3ac41 2024-12-09T12:15:59,892 INFO [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/info/9286a7f456714792a95bfc1814d3ac41, entries=273, sequenceid=64, filesize=35.5 K 2024-12-09T12:15:59,895 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/.tmp/ns/3c2e522857e34724ab2b4fdcca44dbe8 as hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/ns/3c2e522857e34724ab2b4fdcca44dbe8 2024-12-09T12:15:59,904 INFO [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/ns/3c2e522857e34724ab2b4fdcca44dbe8, entries=2, sequenceid=64, filesize=5.0 K 2024-12-09T12:15:59,905 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/.tmp/table/1a1c62dd453048468bb3cb207f9dd13e as hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/table/1a1c62dd453048468bb3cb207f9dd13e 2024-12-09T12:15:59,916 INFO [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/table/1a1c62dd453048468bb3cb207f9dd13e, entries=2, sequenceid=64, filesize=5.0 K 2024-12-09T12:15:59,922 INFO [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(3140): Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 352ms, sequenceid=64, compaction requested=false; wal=null 2024-12-09T12:15:59,924 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-12-09T12:15:59,926 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T12:15:59,926 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T12:15:59,927 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T12:15:59,929 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T12:15:59,931 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/recovered.edits/64.seqid, newMaxSeqId=64, maxSeqId=1 2024-12-09T12:15:59,932 INFO [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=65; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66354729, jitterRate=-0.011237487196922302}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T12:15:59,933 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-09T12:15:59,933 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733746559512Writing region info on filesystem at 1733746559512Initializing all the Stores at 1733746559514 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733746559514Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733746559517 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746559517Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733746559517Obtaining lock to block concurrent updates at 1733746559569 (+52 ms)Preparing flush snapshotting stores in 1588230740 at 1733746559569Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=30111, getHeapSize=51824, getOffHeapSize=0, getCellsCount=277 at 1733746559573 (+4 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733746559573Flushing 1588230740/info: creating writer at 1733746559575 (+2 ms)Flushing 1588230740/info: appending metadata at 1733746559644 (+69 ms)Flushing 1588230740/info: closing flushed file at 1733746559648 (+4 ms)Flushing 1588230740/ns: creating writer at 1733746559791 (+143 ms)Flushing 1588230740/ns: appending metadata at 1733746559808 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1733746559808Flushing 1588230740/table: creating writer at 1733746559836 (+28 ms)Flushing 1588230740/table: appending metadata at 1733746559859 (+23 ms)Flushing 1588230740/table: closing flushed file at 1733746559859Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f85a4f0: reopening flushed file at 1733746559881 (+22 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@90ad98f: reopening flushed file at 1733746559894 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@102e5dfa: reopening flushed file at 1733746559904 (+10 ms)Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 352ms, sequenceid=64, compaction requested=false; wal=null at 1733746559922 (+18 ms)Cleaning up temporary data from old regions at 1733746559926 (+4 ms)Running coprocessor post-open hooks at 1733746559933 (+7 ms)Region opened successfully at 1733746559933 2024-12-09T12:15:59,934 INFO [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=64, masterSystemTime=1733746559457 2024-12-09T12:15:59,936 DEBUG [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-09T12:15:59,936 INFO [RS_OPEN_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-09T12:15:59,937 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=65, regionLocation=f15dd41c88ad,38111,1733746548467 2024-12-09T12:15:59,938 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f15dd41c88ad,38111,1733746548467, state=OPEN 2024-12-09T12:15:59,970 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f15dd41c88ad,38111,1733746548467, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-09T12:16:00,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:00,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:00,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:00,054 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=64, ppid=63, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:00,054 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:00,054 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:00,054 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:00,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-12-09T12:16:00,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 in 750 msec 2024-12-09T12:16:00,059 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=60 2024-12-09T12:16:00,059 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 911 msec 2024-12-09T12:16:00,060 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): f15dd41c88ad,33783,1733746548634 had 9 regions 2024-12-09T12:16:00,061 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true, isMeta: false 2024-12-09T12:16:00,064 INFO [PEWorker-1 {}] master.SplitWALManager(105): f15dd41c88ad,33783,1733746548634 WAL count=1, meta=false 2024-12-09T12:16:00,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223}] 2024-12-09T12:16:00,066 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:00,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727}] 2024-12-09T12:16:00,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36579 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=66 2024-12-09T12:16:00,246 INFO [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(299): Splitting hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634-splitting/f15dd41c88ad%2C33783%2C1733746548634.1733746551223, size=2.7 K (2774bytes) 2024-12-09T12:16:00,246 INFO [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-1 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634-splitting/f15dd41c88ad%2C33783%2C1733746548634.1733746551223 2024-12-09T12:16:00,247 INFO [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-1 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634-splitting/f15dd41c88ad%2C33783%2C1733746548634.1733746551223 after 1ms 2024-12-09T12:16:00,251 INFO [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(310): Open hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634-splitting/f15dd41c88ad%2C33783%2C1733746548634.1733746551223 took 6ms 2024-12-09T12:16:00,256 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 8ceebab6d8b44ccd74ad65e871405953: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-09T12:16:00,258 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for c8f8bc12489e5c0370ba59c751783a23: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-09T12:16:00,261 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for a258d2ecd9a494e8d56406b774da28ef: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-09T12:16:00,263 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for ebee1500f62faadfac8720740a1e527e: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-09T12:16:00,268 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for ba4a26e129719340e5696ac56c6b61ac: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-09T12:16:00,270 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for e5251eee3971dde9f0f276cfd8632c21: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-09T12:16:00,272 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 62a60a0dff0c026a110e755e6c249001: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-09T12:16:00,274 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 43f38f304815b66d00adc6292e217f1a: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-09T12:16:00,276 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 8bc29ffded7f1b74a1ee2986c5e611f8: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-09T12:16:00,276 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634-splitting/f15dd41c88ad%2C33783%2C1733746548634.1733746551223 so closing down 2024-12-09T12:16:00,276 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T12:16:00,276 INFO [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T12:16:00,276 INFO [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(425): Processed 10 edits across 0 Regions in 24 ms; skipped=10; WAL=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634-splitting/f15dd41c88ad%2C33783%2C1733746548634.1733746551223, size=2.7 K, length=2774, corrupted=false, cancelled=false 2024-12-09T12:16:00,277 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(428): Completed split of hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634-splitting/f15dd41c88ad%2C33783%2C1733746548634.1733746551223, journal: Splitting hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634-splitting/f15dd41c88ad%2C33783%2C1733746548634.1733746551223, size=2.7 K (2774bytes) at 1733746560246Finishing writing output for hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634-splitting/f15dd41c88ad%2C33783%2C1733746548634.1733746551223 so closing down at 1733746560276 (+30 ms)3 split writer threads finished at 1733746560276Processed 10 edits across 0 Regions in 24 ms; skipped=10; WAL=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634-splitting/f15dd41c88ad%2C33783%2C1733746548634.1733746551223, size=2.7 K, length=2774, corrupted=false, cancelled=false at 1733746560276 2024-12-09T12:16:00,277 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-1 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634-splitting/f15dd41c88ad%2C33783%2C1733746548634.1733746551223 2024-12-09T12:16:00,277 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f15dd41c88ad:0-1 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-12-09T12:16:00,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42519 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-12-09T12:16:00,289 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634-splitting/f15dd41c88ad%2C33783%2C1733746548634.1733746551223 to hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/oldWALs 2024-12-09T12:16:00,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-12-09T12:16:00,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 in 223 msec 2024-12-09T12:16:00,327 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:00,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=60 2024-12-09T12:16:00,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 in 263 msec 2024-12-09T12:16:00,352 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/WALs/f15dd41c88ad,33783,1733746548634-splitting dir is empty, no logs to split. 2024-12-09T12:16:00,353 INFO [PEWorker-1 {}] master.SplitWALManager(105): f15dd41c88ad,33783,1733746548634 WAL count=0, meta=false 2024-12-09T12:16:00,353 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if f15dd41c88ad,33783,1733746548634 WAL splitting is done? wals=0, meta=false 2024-12-09T12:16:00,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN}, {pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN}, {pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN}, {pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN}, {pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN}, {pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN}, {pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN}, {pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN}] 2024-12-09T12:16:00,360 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN 2024-12-09T12:16:00,364 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-09T12:16:00,371 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN 2024-12-09T12:16:00,372 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-09T12:16:00,373 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN 2024-12-09T12:16:00,374 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-09T12:16:00,375 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN 2024-12-09T12:16:00,377 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-09T12:16:00,378 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN 2024-12-09T12:16:00,379 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-09T12:16:00,380 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:00,381 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-09T12:16:00,383 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:00,384 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-09T12:16:00,391 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:00,393 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-09T12:16:00,515 DEBUG [f15dd41c88ad:42519 {}] balancer.BalancerClusterState(204): Hosts are {f15dd41c88ad=0} racks are {/default-rack=0} 2024-12-09T12:16:00,515 DEBUG [f15dd41c88ad:42519 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T12:16:00,515 DEBUG [f15dd41c88ad:42519 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T12:16:00,516 DEBUG [f15dd41c88ad:42519 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T12:16:00,516 DEBUG [f15dd41c88ad:42519 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T12:16:00,516 INFO [f15dd41c88ad:42519 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T12:16:00,516 INFO [f15dd41c88ad:42519 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T12:16:00,516 DEBUG [f15dd41c88ad:42519 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T12:16:00,516 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=c8f8bc12489e5c0370ba59c751783a23, regionState=OPENING, regionLocation=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:00,517 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=ba4a26e129719340e5696ac56c6b61ac, regionState=OPENING, regionLocation=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:00,518 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=62a60a0dff0c026a110e755e6c249001, regionState=OPENING, regionLocation=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:00,518 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=a258d2ecd9a494e8d56406b774da28ef, regionState=OPENING, regionLocation=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:00,518 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f15dd41c88ad:33783 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f15dd41c88ad/172.17.0.3:33783 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T12:16:00,519 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=8ceebab6d8b44ccd74ad65e871405953, regionState=OPENING, regionLocation=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:00,519 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=43f38f304815b66d00adc6292e217f1a, regionState=OPENING, regionLocation=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:00,520 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=ebee1500f62faadfac8720740a1e527e, regionState=OPENING, regionLocation=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:00,520 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=8bc29ffded7f1b74a1ee2986c5e611f8, regionState=OPENING, regionLocation=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:00,520 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f15dd41c88ad,33783,1733746548634, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f15dd41c88ad,33783,1733746548634, seqNum=-1, error=java.net.ConnectException: Call to address=f15dd41c88ad:33783 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f15dd41c88ad/172.17.0.3:33783 2024-12-09T12:16:00,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f15dd41c88ad,33783,1733746548634, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-09T12:16:00,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f15dd41c88ad,33783,1733746548634, seqNum=-1 from cache 2024-12-09T12:16:00,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f15dd41c88ad,33783,1733746548634, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=f15dd41c88ad:33783 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f15dd41c88ad/172.17.0.3:33783 2024-12-09T12:16:00,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address f15dd41c88ad:33783 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f15dd41c88ad/172.17.0.3:33783 2024-12-09T12:16:00,523 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f15dd41c88ad:33783 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f15dd41c88ad/172.17.0.3:33783 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T12:16:00,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f15dd41c88ad,33783,1733746548634, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=f15dd41c88ad:33783 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f15dd41c88ad/172.17.0.3:33783 2024-12-09T12:16:00,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f15dd41c88ad,33783,1733746548634, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=f15dd41c88ad:33783 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f15dd41c88ad/172.17.0.3:33783 2024-12-09T12:16:00,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f15dd41c88ad,33783,1733746548634, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=f15dd41c88ad:33783 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f15dd41c88ad/172.17.0.3:33783 2024-12-09T12:16:00,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f15dd41c88ad,33783,1733746548634, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=f15dd41c88ad:33783 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f15dd41c88ad/172.17.0.3:33783 2024-12-09T12:16:00,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f15dd41c88ad,33783,1733746548634, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=f15dd41c88ad:33783 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f15dd41c88ad/172.17.0.3:33783 2024-12-09T12:16:00,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f15dd41c88ad,33783,1733746548634, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=f15dd41c88ad:33783 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f15dd41c88ad/172.17.0.3:33783 2024-12-09T12:16:00,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address f15dd41c88ad:33783 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f15dd41c88ad/172.17.0.3:33783 2024-12-09T12:16:00,632 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T12:16:00,632 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f15dd41c88ad,38111,1733746548467, seqNum=-1] 2024-12-09T12:16:00,632 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T12:16:00,634 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60361, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T12:16:00,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN because future has completed 2024-12-09T12:16:00,639 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN because future has completed 2024-12-09T12:16:00,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba4a26e129719340e5696ac56c6b61ac, server=f15dd41c88ad,38111,1733746548467}] 2024-12-09T12:16:00,640 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN because future has completed 2024-12-09T12:16:00,641 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN because future has completed 2024-12-09T12:16:00,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN because future has completed 2024-12-09T12:16:00,645 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN because future has completed 2024-12-09T12:16:00,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8bc29ffded7f1b74a1ee2986c5e611f8, server=f15dd41c88ad,36579,1733746548727}] 2024-12-09T12:16:00,647 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN because future has completed 2024-12-09T12:16:00,649 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN because future has completed 2024-12-09T12:16:00,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a258d2ecd9a494e8d56406b774da28ef, server=f15dd41c88ad,38111,1733746548467}] 2024-12-09T12:16:00,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467}] 2024-12-09T12:16:00,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure ebee1500f62faadfac8720740a1e527e, server=f15dd41c88ad,36579,1733746548727}] 2024-12-09T12:16:00,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43f38f304815b66d00adc6292e217f1a, server=f15dd41c88ad,36579,1733746548727}] 2024-12-09T12:16:00,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ceebab6d8b44ccd74ad65e871405953, server=f15dd41c88ad,36579,1733746548727}] 2024-12-09T12:16:00,694 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8f8bc12489e5c0370ba59c751783a23, server=f15dd41c88ad,38111,1733746548467}] 2024-12-09T12:16:00,797 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,mmm,1733746552866.62a60a0dff0c026a110e755e6c249001. 2024-12-09T12:16:00,797 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 62a60a0dff0c026a110e755e6c249001, NAME => 'test,mmm,1733746552866.62a60a0dff0c026a110e755e6c249001.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-12-09T12:16:00,798 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 62a60a0dff0c026a110e755e6c249001 2024-12-09T12:16:00,798 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,mmm,1733746552866.62a60a0dff0c026a110e755e6c249001.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:16:00,798 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 62a60a0dff0c026a110e755e6c249001 2024-12-09T12:16:00,798 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 62a60a0dff0c026a110e755e6c249001 2024-12-09T12:16:00,800 INFO [StoreOpener-62a60a0dff0c026a110e755e6c249001-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 62a60a0dff0c026a110e755e6c249001 2024-12-09T12:16:00,801 INFO [StoreOpener-62a60a0dff0c026a110e755e6c249001-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62a60a0dff0c026a110e755e6c249001 columnFamilyName family 2024-12-09T12:16:00,801 DEBUG [StoreOpener-62a60a0dff0c026a110e755e6c249001-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:16:00,802 INFO [StoreOpener-62a60a0dff0c026a110e755e6c249001-1 {}] regionserver.HStore(327): Store=62a60a0dff0c026a110e755e6c249001/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:16:00,802 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 62a60a0dff0c026a110e755e6c249001 2024-12-09T12:16:00,803 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/62a60a0dff0c026a110e755e6c249001 2024-12-09T12:16:00,805 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open test,www,1733746552866.8ceebab6d8b44ccd74ad65e871405953. 2024-12-09T12:16:00,805 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => 8ceebab6d8b44ccd74ad65e871405953, NAME => 'test,www,1733746552866.8ceebab6d8b44ccd74ad65e871405953.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-12-09T12:16:00,805 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/62a60a0dff0c026a110e755e6c249001 2024-12-09T12:16:00,805 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8ceebab6d8b44ccd74ad65e871405953 2024-12-09T12:16:00,806 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated test,www,1733746552866.8ceebab6d8b44ccd74ad65e871405953.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:16:00,806 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for 8ceebab6d8b44ccd74ad65e871405953 2024-12-09T12:16:00,806 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for 8ceebab6d8b44ccd74ad65e871405953 2024-12-09T12:16:00,806 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 62a60a0dff0c026a110e755e6c249001 2024-12-09T12:16:00,806 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 62a60a0dff0c026a110e755e6c249001 2024-12-09T12:16:00,808 INFO [StoreOpener-8ceebab6d8b44ccd74ad65e871405953-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8ceebab6d8b44ccd74ad65e871405953 2024-12-09T12:16:00,812 INFO [StoreOpener-8ceebab6d8b44ccd74ad65e871405953-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ceebab6d8b44ccd74ad65e871405953 columnFamilyName family 2024-12-09T12:16:00,812 DEBUG [StoreOpener-8ceebab6d8b44ccd74ad65e871405953-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:16:00,813 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 62a60a0dff0c026a110e755e6c249001 2024-12-09T12:16:00,813 INFO [StoreOpener-8ceebab6d8b44ccd74ad65e871405953-1 {}] regionserver.HStore(327): Store=8ceebab6d8b44ccd74ad65e871405953/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:16:00,813 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for 8ceebab6d8b44ccd74ad65e871405953 2024-12-09T12:16:00,815 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/8ceebab6d8b44ccd74ad65e871405953 2024-12-09T12:16:00,815 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 62a60a0dff0c026a110e755e6c249001; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67449054, jitterRate=0.005069226026535034}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:16:00,815 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 62a60a0dff0c026a110e755e6c249001 2024-12-09T12:16:00,815 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 62a60a0dff0c026a110e755e6c249001: Running coprocessor pre-open hook at 1733746560798Writing region info on filesystem at 1733746560798Initializing all the Stores at 1733746560799 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746560799Cleaning up temporary data from old regions at 1733746560806 (+7 ms)Running coprocessor post-open hooks at 1733746560815 (+9 ms)Region opened successfully at 1733746560815 2024-12-09T12:16:00,816 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1733746552866.62a60a0dff0c026a110e755e6c249001., pid=78, masterSystemTime=1733746560792 2024-12-09T12:16:00,816 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/8ceebab6d8b44ccd74ad65e871405953 2024-12-09T12:16:00,817 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for 8ceebab6d8b44ccd74ad65e871405953 2024-12-09T12:16:00,817 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for 8ceebab6d8b44ccd74ad65e871405953 2024-12-09T12:16:00,819 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for 8ceebab6d8b44ccd74ad65e871405953 2024-12-09T12:16:00,820 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened 8ceebab6d8b44ccd74ad65e871405953; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65110416, jitterRate=-0.02977919578552246}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:16:00,820 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8ceebab6d8b44ccd74ad65e871405953 2024-12-09T12:16:00,820 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for 8ceebab6d8b44ccd74ad65e871405953: Running coprocessor pre-open hook at 1733746560806Writing region info on filesystem at 1733746560806Initializing all the Stores at 1733746560807 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746560807Cleaning up temporary data from old regions at 1733746560817 (+10 ms)Running coprocessor post-open hooks at 1733746560820 (+3 ms)Region opened successfully at 1733746560820 2024-12-09T12:16:00,823 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1733746552866.8ceebab6d8b44ccd74ad65e871405953., pid=81, masterSystemTime=1733746560800 2024-12-09T12:16:00,823 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1733746552866.62a60a0dff0c026a110e755e6c249001. 2024-12-09T12:16:00,823 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,mmm,1733746552866.62a60a0dff0c026a110e755e6c249001. 2024-12-09T12:16:00,823 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,ttt,1733746552866.ba4a26e129719340e5696ac56c6b61ac. 2024-12-09T12:16:00,823 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => ba4a26e129719340e5696ac56c6b61ac, NAME => 'test,ttt,1733746552866.ba4a26e129719340e5696ac56c6b61ac.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-12-09T12:16:00,824 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ba4a26e129719340e5696ac56c6b61ac 2024-12-09T12:16:00,824 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,ttt,1733746552866.ba4a26e129719340e5696ac56c6b61ac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:16:00,824 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for ba4a26e129719340e5696ac56c6b61ac 2024-12-09T12:16:00,824 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for ba4a26e129719340e5696ac56c6b61ac 2024-12-09T12:16:00,826 INFO [StoreOpener-ba4a26e129719340e5696ac56c6b61ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ba4a26e129719340e5696ac56c6b61ac 2024-12-09T12:16:00,827 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-12-09T12:16:00,827 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1733746552866.8ceebab6d8b44ccd74ad65e871405953. 2024-12-09T12:16:00,827 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened test,www,1733746552866.8ceebab6d8b44ccd74ad65e871405953. 2024-12-09T12:16:00,827 INFO [StoreOpener-ba4a26e129719340e5696ac56c6b61ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba4a26e129719340e5696ac56c6b61ac columnFamilyName family 2024-12-09T12:16:00,827 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:00,827 DEBUG [StoreOpener-ba4a26e129719340e5696ac56c6b61ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:16:00,827 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,lll,1733746552866.ebee1500f62faadfac8720740a1e527e. 2024-12-09T12:16:00,827 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => ebee1500f62faadfac8720740a1e527e, NAME => 'test,lll,1733746552866.ebee1500f62faadfac8720740a1e527e.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-12-09T12:16:00,828 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ebee1500f62faadfac8720740a1e527e 2024-12-09T12:16:00,828 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,lll,1733746552866.ebee1500f62faadfac8720740a1e527e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:16:00,828 INFO [StoreOpener-ba4a26e129719340e5696ac56c6b61ac-1 {}] regionserver.HStore(327): Store=ba4a26e129719340e5696ac56c6b61ac/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:16:00,828 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for ebee1500f62faadfac8720740a1e527e 2024-12-09T12:16:00,828 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for ba4a26e129719340e5696ac56c6b61ac 2024-12-09T12:16:00,828 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for ebee1500f62faadfac8720740a1e527e 2024-12-09T12:16:00,827 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:00,829 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/ba4a26e129719340e5696ac56c6b61ac 2024-12-09T12:16:00,829 INFO [StoreOpener-ebee1500f62faadfac8720740a1e527e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ebee1500f62faadfac8720740a1e527e 2024-12-09T12:16:00,830 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/ba4a26e129719340e5696ac56c6b61ac 2024-12-09T12:16:00,831 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for ba4a26e129719340e5696ac56c6b61ac 2024-12-09T12:16:00,831 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for ba4a26e129719340e5696ac56c6b61ac 2024-12-09T12:16:00,831 INFO [StoreOpener-ebee1500f62faadfac8720740a1e527e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ebee1500f62faadfac8720740a1e527e columnFamilyName family 2024-12-09T12:16:00,831 DEBUG [StoreOpener-ebee1500f62faadfac8720740a1e527e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:16:00,832 INFO [StoreOpener-ebee1500f62faadfac8720740a1e527e-1 {}] regionserver.HStore(327): Store=ebee1500f62faadfac8720740a1e527e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:16:00,832 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for ebee1500f62faadfac8720740a1e527e 2024-12-09T12:16:00,833 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/ebee1500f62faadfac8720740a1e527e 2024-12-09T12:16:00,833 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for ba4a26e129719340e5696ac56c6b61ac 2024-12-09T12:16:00,834 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/ebee1500f62faadfac8720740a1e527e 2024-12-09T12:16:00,834 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened ba4a26e129719340e5696ac56c6b61ac; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60114038, jitterRate=-0.10423102974891663}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:16:00,835 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ba4a26e129719340e5696ac56c6b61ac 2024-12-09T12:16:00,835 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for ba4a26e129719340e5696ac56c6b61ac: Running coprocessor pre-open hook at 1733746560824Writing region info on filesystem at 1733746560824Initializing all the Stores at 1733746560825 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746560825Cleaning up temporary data from old regions at 1733746560831 (+6 ms)Running coprocessor post-open hooks at 1733746560835 (+4 ms)Region opened successfully at 1733746560835 2024-12-09T12:16:00,835 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for ebee1500f62faadfac8720740a1e527e 2024-12-09T12:16:00,835 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for ebee1500f62faadfac8720740a1e527e 2024-12-09T12:16:00,836 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1733746552866.ba4a26e129719340e5696ac56c6b61ac., pid=75, masterSystemTime=1733746560792 2024-12-09T12:16:00,839 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for ebee1500f62faadfac8720740a1e527e 2024-12-09T12:16:00,840 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1733746552866.ba4a26e129719340e5696ac56c6b61ac. 2024-12-09T12:16:00,840 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,ttt,1733746552866.ba4a26e129719340e5696ac56c6b61ac. 2024-12-09T12:16:00,840 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,hhh,1733746552866.a258d2ecd9a494e8d56406b774da28ef. 2024-12-09T12:16:00,840 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => a258d2ecd9a494e8d56406b774da28ef, NAME => 'test,hhh,1733746552866.a258d2ecd9a494e8d56406b774da28ef.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-12-09T12:16:00,841 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a258d2ecd9a494e8d56406b774da28ef 2024-12-09T12:16:00,841 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,hhh,1733746552866.a258d2ecd9a494e8d56406b774da28ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:16:00,841 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for a258d2ecd9a494e8d56406b774da28ef 2024-12-09T12:16:00,841 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for a258d2ecd9a494e8d56406b774da28ef 2024-12-09T12:16:00,842 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened ebee1500f62faadfac8720740a1e527e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68174044, jitterRate=0.015872418880462646}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:16:00,842 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ebee1500f62faadfac8720740a1e527e 2024-12-09T12:16:00,842 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for ebee1500f62faadfac8720740a1e527e: Running coprocessor pre-open hook at 1733746560828Writing region info on filesystem at 1733746560828Initializing all the Stores at 1733746560829 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746560829Cleaning up temporary data from old regions at 1733746560835 (+6 ms)Running coprocessor post-open hooks at 1733746560842 (+7 ms)Region opened successfully at 1733746560842 2024-12-09T12:16:00,843 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1733746552866.ebee1500f62faadfac8720740a1e527e., pid=79, masterSystemTime=1733746560800 2024-12-09T12:16:00,843 INFO [StoreOpener-a258d2ecd9a494e8d56406b774da28ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a258d2ecd9a494e8d56406b774da28ef 2024-12-09T12:16:00,845 INFO [StoreOpener-a258d2ecd9a494e8d56406b774da28ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a258d2ecd9a494e8d56406b774da28ef columnFamilyName family 2024-12-09T12:16:00,845 DEBUG [StoreOpener-a258d2ecd9a494e8d56406b774da28ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:16:00,845 INFO [StoreOpener-a258d2ecd9a494e8d56406b774da28ef-1 {}] regionserver.HStore(327): Store=a258d2ecd9a494e8d56406b774da28ef/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:16:00,846 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for a258d2ecd9a494e8d56406b774da28ef 2024-12-09T12:16:00,847 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/a258d2ecd9a494e8d56406b774da28ef 2024-12-09T12:16:00,847 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-12-09T12:16:00,849 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/a258d2ecd9a494e8d56406b774da28ef 2024-12-09T12:16:00,850 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1733746552866.ebee1500f62faadfac8720740a1e527e. 2024-12-09T12:16:00,850 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-09T12:16:00,850 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,lll,1733746552866.ebee1500f62faadfac8720740a1e527e. 2024-12-09T12:16:00,850 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=82, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8f8bc12489e5c0370ba59c751783a23, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:00,850 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for a258d2ecd9a494e8d56406b774da28ef 2024-12-09T12:16:00,850 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:00,850 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open test,ppp,1733746552866.43f38f304815b66d00adc6292e217f1a. 2024-12-09T12:16:00,850 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for a258d2ecd9a494e8d56406b774da28ef 2024-12-09T12:16:00,850 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => 43f38f304815b66d00adc6292e217f1a, NAME => 'test,ppp,1733746552866.43f38f304815b66d00adc6292e217f1a.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-12-09T12:16:00,850 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 43f38f304815b66d00adc6292e217f1a 2024-12-09T12:16:00,851 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated test,ppp,1733746552866.43f38f304815b66d00adc6292e217f1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:16:00,851 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for 43f38f304815b66d00adc6292e217f1a 2024-12-09T12:16:00,851 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for 43f38f304815b66d00adc6292e217f1a 2024-12-09T12:16:00,858 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=82, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8f8bc12489e5c0370ba59c751783a23, server=f15dd41c88ad,38111,1733746548467 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:00,862 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for a258d2ecd9a494e8d56406b774da28ef 2024-12-09T12:16:00,863 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened a258d2ecd9a494e8d56406b774da28ef; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68538283, jitterRate=0.021300002932548523}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:16:00,864 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a258d2ecd9a494e8d56406b774da28ef 2024-12-09T12:16:00,864 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for a258d2ecd9a494e8d56406b774da28ef: Running coprocessor pre-open hook at 1733746560841Writing region info on filesystem at 1733746560841Initializing all the Stores at 1733746560842 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746560842Cleaning up temporary data from old regions at 1733746560850 (+8 ms)Running coprocessor post-open hooks at 1733746560864 (+14 ms)Region opened successfully at 1733746560864 2024-12-09T12:16:00,865 INFO [StoreOpener-43f38f304815b66d00adc6292e217f1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 43f38f304815b66d00adc6292e217f1a 2024-12-09T12:16:00,865 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1733746552866.a258d2ecd9a494e8d56406b774da28ef., pid=77, masterSystemTime=1733746560792 2024-12-09T12:16:00,869 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1733746552866.a258d2ecd9a494e8d56406b774da28ef. 2024-12-09T12:16:00,869 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,hhh,1733746552866.a258d2ecd9a494e8d56406b774da28ef. 2024-12-09T12:16:00,869 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(132): Open test,zzz,1733746552866.c8f8bc12489e5c0370ba59c751783a23. 2024-12-09T12:16:00,869 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7752): Opening region: {ENCODED => c8f8bc12489e5c0370ba59c751783a23, NAME => 'test,zzz,1733746552866.c8f8bc12489e5c0370ba59c751783a23.', STARTKEY => 'zzz', ENDKEY => ''} 2024-12-09T12:16:00,870 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c8f8bc12489e5c0370ba59c751783a23 2024-12-09T12:16:00,870 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(898): Instantiated test,zzz,1733746552866.c8f8bc12489e5c0370ba59c751783a23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:16:00,870 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7794): checking encryption for c8f8bc12489e5c0370ba59c751783a23 2024-12-09T12:16:00,870 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7797): checking classloading for c8f8bc12489e5c0370ba59c751783a23 2024-12-09T12:16:00,871 INFO [StoreOpener-43f38f304815b66d00adc6292e217f1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43f38f304815b66d00adc6292e217f1a columnFamilyName family 2024-12-09T12:16:00,871 DEBUG [StoreOpener-43f38f304815b66d00adc6292e217f1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:16:00,872 INFO [StoreOpener-43f38f304815b66d00adc6292e217f1a-1 {}] regionserver.HStore(327): Store=43f38f304815b66d00adc6292e217f1a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:16:00,872 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for 43f38f304815b66d00adc6292e217f1a 2024-12-09T12:16:00,872 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/43f38f304815b66d00adc6292e217f1a 2024-12-09T12:16:00,874 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/43f38f304815b66d00adc6292e217f1a 2024-12-09T12:16:00,874 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for 43f38f304815b66d00adc6292e217f1a 2024-12-09T12:16:00,874 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for 43f38f304815b66d00adc6292e217f1a 2024-12-09T12:16:00,877 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for 43f38f304815b66d00adc6292e217f1a 2024-12-09T12:16:00,878 INFO [StoreOpener-c8f8bc12489e5c0370ba59c751783a23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c8f8bc12489e5c0370ba59c751783a23 2024-12-09T12:16:00,879 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened 43f38f304815b66d00adc6292e217f1a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58910180, jitterRate=-0.12216991186141968}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:16:00,879 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 43f38f304815b66d00adc6292e217f1a 2024-12-09T12:16:00,879 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for 43f38f304815b66d00adc6292e217f1a: Running coprocessor pre-open hook at 1733746560851Writing region info on filesystem at 1733746560851Initializing all the Stores at 1733746560861 (+10 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746560861Cleaning up temporary data from old regions at 1733746560874 (+13 ms)Running coprocessor post-open hooks at 1733746560879 (+5 ms)Region opened successfully at 1733746560879 2024-12-09T12:16:00,881 INFO [StoreOpener-c8f8bc12489e5c0370ba59c751783a23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8f8bc12489e5c0370ba59c751783a23 columnFamilyName family 2024-12-09T12:16:00,881 DEBUG [StoreOpener-c8f8bc12489e5c0370ba59c751783a23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:16:00,881 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1733746552866.43f38f304815b66d00adc6292e217f1a., pid=80, masterSystemTime=1733746560800 2024-12-09T12:16:00,882 INFO [StoreOpener-c8f8bc12489e5c0370ba59c751783a23-1 {}] regionserver.HStore(327): Store=c8f8bc12489e5c0370ba59c751783a23/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:16:00,882 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1038): replaying wal for c8f8bc12489e5c0370ba59c751783a23 2024-12-09T12:16:00,883 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/c8f8bc12489e5c0370ba59c751783a23 2024-12-09T12:16:00,885 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1733746552866.43f38f304815b66d00adc6292e217f1a. 2024-12-09T12:16:00,885 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened test,ppp,1733746552866.43f38f304815b66d00adc6292e217f1a. 2024-12-09T12:16:00,885 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,ccc,1733746552866.8bc29ffded7f1b74a1ee2986c5e611f8. 2024-12-09T12:16:00,885 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 8bc29ffded7f1b74a1ee2986c5e611f8, NAME => 'test,ccc,1733746552866.8bc29ffded7f1b74a1ee2986c5e611f8.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-12-09T12:16:00,886 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8bc29ffded7f1b74a1ee2986c5e611f8 2024-12-09T12:16:00,886 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,ccc,1733746552866.8bc29ffded7f1b74a1ee2986c5e611f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T12:16:00,886 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 8bc29ffded7f1b74a1ee2986c5e611f8 2024-12-09T12:16:00,886 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 8bc29ffded7f1b74a1ee2986c5e611f8 2024-12-09T12:16:00,888 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-09T12:16:00,888 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:00,889 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T12:16:00,889 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T12:16:00,890 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-09T12:16:00,890 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-09T12:16:00,890 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-09T12:16:00,890 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/c8f8bc12489e5c0370ba59c751783a23 2024-12-09T12:16:00,890 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T12:16:00,895 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 4 msec 2024-12-09T12:16:00,895 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1048): stopping wal replay for c8f8bc12489e5c0370ba59c751783a23 2024-12-09T12:16:00,895 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1060): Cleaning up temporary data for c8f8bc12489e5c0370ba59c751783a23 2024-12-09T12:16:00,897 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1093): writing seq id for c8f8bc12489e5c0370ba59c751783a23 2024-12-09T12:16:00,899 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1114): Opened c8f8bc12489e5c0370ba59c751783a23; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59934726, jitterRate=-0.10690298676490784}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:16:00,899 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c8f8bc12489e5c0370ba59c751783a23 2024-12-09T12:16:00,899 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1006): Region open journal for c8f8bc12489e5c0370ba59c751783a23: Running coprocessor pre-open hook at 1733746560870Writing region info on filesystem at 1733746560870Initializing all the Stores at 1733746560871 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746560871Cleaning up temporary data from old regions at 1733746560895 (+24 ms)Running coprocessor post-open hooks at 1733746560899 (+4 ms)Region opened successfully at 1733746560899 2024-12-09T12:16:00,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-09T12:16:00,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:00,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:00,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:00,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:00,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:00,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:00,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:00,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:00,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:00,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:00,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:00,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=82, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:00,903 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1733746552866.c8f8bc12489e5c0370ba59c751783a23., pid=82, masterSystemTime=1733746560792 2024-12-09T12:16:00,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:00,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:00,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:00,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:00,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:00,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:00,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:00,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:00,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:00,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:00,903 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:00,904 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:00,904 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T12:16:00,906 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-09T12:16:00,906 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "f15dd41c88ad" port: 38111 start_code: 1733746548467 } transition { transition_code: OPENED region_info { region_id: 1733746552866 table_name { namespace: "default" qualifier: "test" } start_key: "zzz" end_key: "" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 82 initiating_master_active_time: 1733746548918 }; retry (#0) after 1008ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-12-09T12:16:00,906 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, REOPEN/MOVE 2024-12-09T12:16:00,909 INFO [StoreOpener-8bc29ffded7f1b74a1ee2986c5e611f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8bc29ffded7f1b74a1ee2986c5e611f8 2024-12-09T12:16:00,911 INFO [StoreOpener-8bc29ffded7f1b74a1ee2986c5e611f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8bc29ffded7f1b74a1ee2986c5e611f8 columnFamilyName family 2024-12-09T12:16:00,911 DEBUG [StoreOpener-8bc29ffded7f1b74a1ee2986c5e611f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T12:16:00,912 INFO [StoreOpener-8bc29ffded7f1b74a1ee2986c5e611f8-1 {}] regionserver.HStore(327): Store=8bc29ffded7f1b74a1ee2986c5e611f8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T12:16:00,912 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true 2024-12-09T12:16:00,912 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:00,912 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:00,912 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T12:16:00,912 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:00,912 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:00,912 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:00,912 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:00,912 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 8bc29ffded7f1b74a1ee2986c5e611f8 2024-12-09T12:16:00,912 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:00,912 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:00,912 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN 2024-12-09T12:16:00,912 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN 2024-12-09T12:16:00,912 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN 2024-12-09T12:16:00,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN 2024-12-09T12:16:00,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN 2024-12-09T12:16:00,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba4a26e129719340e5696ac56c6b61ac, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:00,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8bc29ffded7f1b74a1ee2986c5e611f8, server=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:00,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a258d2ecd9a494e8d56406b774da28ef, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:00,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:00,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure ebee1500f62faadfac8720740a1e527e, server=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:00,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43f38f304815b66d00adc6292e217f1a, server=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:00,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ceebab6d8b44ccd74ad65e871405953, server=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:00,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=82, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8f8bc12489e5c0370ba59c751783a23, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:00,913 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:00,913 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:00,913 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:00,913 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:00,913 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/8bc29ffded7f1b74a1ee2986c5e611f8 2024-12-09T12:16:00,913 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:00,913 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:00,913 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:00,913 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:00,913 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:00,914 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:00,914 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:00,914 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:00,914 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:00,914 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:00,914 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN 2024-12-09T12:16:00,914 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:00,914 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN 2024-12-09T12:16:00,914 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:00,914 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN 2024-12-09T12:16:00,914 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:00,914 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN 2024-12-09T12:16:00,914 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:00,914 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN 2024-12-09T12:16:00,914 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba4a26e129719340e5696ac56c6b61ac, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:00,914 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8bc29ffded7f1b74a1ee2986c5e611f8, server=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:00,914 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a258d2ecd9a494e8d56406b774da28ef, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:00,914 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:00,914 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure ebee1500f62faadfac8720740a1e527e, server=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:00,914 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43f38f304815b66d00adc6292e217f1a, server=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:00,914 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ceebab6d8b44ccd74ad65e871405953, server=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:00,914 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=82, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8f8bc12489e5c0370ba59c751783a23, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:00,915 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/8bc29ffded7f1b74a1ee2986c5e611f8 2024-12-09T12:16:00,915 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 20 msec 2024-12-09T12:16:00,917 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 8bc29ffded7f1b74a1ee2986c5e611f8 2024-12-09T12:16:00,917 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 8bc29ffded7f1b74a1ee2986c5e611f8 2024-12-09T12:16:00,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=65 2024-12-09T12:16:00,917 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f15dd41c88ad,38111,1733746548467, table=hbase:meta, region=1588230740 2024-12-09T12:16:00,918 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f15dd41c88ad,38111,1733746548467, state=OPEN 2024-12-09T12:16:00,919 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 8bc29ffded7f1b74a1ee2986c5e611f8 2024-12-09T12:16:00,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:00,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:00,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:00,920 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:00,920 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:00,920 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:00,922 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN to state=OFFLINE, location=null, table=test, region=62a60a0dff0c026a110e755e6c249001 to restore RIT 2024-12-09T12:16:00,922 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN to state=OFFLINE, location=null, table=test, region=ba4a26e129719340e5696ac56c6b61ac to restore RIT 2024-12-09T12:16:00,922 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN to state=OFFLINE, location=null, table=test, region=ebee1500f62faadfac8720740a1e527e to restore RIT 2024-12-09T12:16:00,922 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN to state=OFFLINE, location=null, table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8 to restore RIT 2024-12-09T12:16:00,922 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN to state=OFFLINE, location=null, table=test, region=c8f8bc12489e5c0370ba59c751783a23 to restore RIT 2024-12-09T12:16:00,922 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN to state=OFFLINE, location=null, table=test, region=a258d2ecd9a494e8d56406b774da28ef to restore RIT 2024-12-09T12:16:00,922 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN to state=OFFLINE, location=null, table=test, region=8ceebab6d8b44ccd74ad65e871405953 to restore RIT 2024-12-09T12:16:00,922 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN to state=OFFLINE, location=null, table=test, region=43f38f304815b66d00adc6292e217f1a to restore RIT 2024-12-09T12:16:00,923 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 8bc29ffded7f1b74a1ee2986c5e611f8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68108869, jitterRate=0.014901235699653625}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T12:16:00,923 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8bc29ffded7f1b74a1ee2986c5e611f8 2024-12-09T12:16:00,923 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 8bc29ffded7f1b74a1ee2986c5e611f8: Running coprocessor pre-open hook at 1733746560886Writing region info on filesystem at 1733746560886Initializing all the Stores at 1733746560898 (+12 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733746560899 (+1 ms)Cleaning up temporary data from old regions at 1733746560917 (+18 ms)Running coprocessor post-open hooks at 1733746560923 (+6 ms)Region opened successfully at 1733746560923 2024-12-09T12:16:00,925 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1733746552866.8bc29ffded7f1b74a1ee2986c5e611f8., pid=76, masterSystemTime=1733746560800 2024-12-09T12:16:00,927 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1924) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1288) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1214) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1264) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:00,931 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] ipc.CallRunner(138): callId: 118 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 126 connection: 172.17.0.3:37215 deadline: 1733746570926, exception=org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false 2024-12-09T12:16:00,931 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733746590931 2024-12-09T12:16:00,932 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T12:16:00,932 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "f15dd41c88ad" port: 36579 start_code: 1733746548727 } transition { transition_code: OPENED region_info { region_id: 1733746552866 table_name { namespace: "default" qualifier: "test" } start_key: "ccc" end_key: "ddd" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 76 initiating_master_active_time: 1733746548918 }; retry (#0) after 1004ms delay (Master is coming online...). org.apache.hadoop.hbase.PleaseHoldException: org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1924) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1288) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1214) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1264) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1924) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1288) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1214) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1264) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-12-09T12:16:00,932 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:00,933 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-09T12:16:00,935 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-09T12:16:00,935 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=81, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ceebab6d8b44ccd74ad65e871405953, server=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:00,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:00,935 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=81, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ceebab6d8b44ccd74ad65e871405953, server=f15dd41c88ad,36579,1733746548727 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:00,962 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc7aaad28378d11daf90725c0f9d39f5, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:00,962 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ac6650f2a35b2c131e93ead260da106, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:00,962 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8bc29ffded7f1b74a1ee2986c5e611f8, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:00,963 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ce28e8d0ce113d2637ab328f5ffd3be, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:00,963 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5251eee3971dde9f0f276cfd8632c21, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=5 2024-12-09T12:16:00,963 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b5306e854c5aec62eff62a18cec5859, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:00,964 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c910d27fbf3265165c7fd59609e57c82, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:00,964 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a258d2ecd9a494e8d56406b774da28ef, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:00,964 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0cf25c9917186a9e3dbd87e44f33615, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:00,964 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9e23742363329565223433d05d86296, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:00,965 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9702c569de70ef36018a26943c3226f, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:00,965 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebee1500f62faadfac8720740a1e527e, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:00,965 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62a60a0dff0c026a110e755e6c249001, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:00,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdcb3d892a0c9cf641eaedbbd9ec5937, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:00,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a1205a417c23329b514f783f4727022, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:00,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43f38f304815b66d00adc6292e217f1a, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:00,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab7cda743d5cc81945ffb7c431d6346, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:00,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0adbf1ba01f94cad79f2814d407182f, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:00,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603cf0cbb0e068a5f7727e4abab3a5d5, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:00,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba4a26e129719340e5696ac56c6b61ac, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:00,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0351c7e96a16dae18ff5a8d1664b2b1, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:00,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23280f1743db631eaa35bf3487e69147, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:00,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ceebab6d8b44ccd74ad65e871405953, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:00,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a763ac99f0e3a37f94a31b5f0479530, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:00,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eaca6c428d8bff0cd952b3c2dfe5408, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:00,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8f8bc12489e5c0370ba59c751783a23, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:00,968 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-09T12:16:00,968 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733746620968 2024-12-09T12:16:00,968 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733746680968 2024-12-09T12:16:00,968 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 36 msec 2024-12-09T12:16:00,968 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-09T12:16:00,968 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:00,969 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T12:16:00,969 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T12:16:00,969 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-09T12:16:00,969 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-09T12:16:00,969 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-09T12:16:00,969 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T12:16:00,970 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-09T12:16:00,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-09T12:16:00,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:00,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:00,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:00,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:00,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:00,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:00,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:00,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:00,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:00,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:00,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:00,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:00,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:00,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:00,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:00,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:00,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:00,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:00,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:00,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:00,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:00,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T12:16:00,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-09T12:16:00,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, REOPEN/MOVE 2024-12-09T12:16:00,978 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true 2024-12-09T12:16:00,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:00,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:00,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T12:16:00,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:00,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:00,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:00,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:00,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:00,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:00,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN 2024-12-09T12:16:00,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN 2024-12-09T12:16:00,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN 2024-12-09T12:16:00,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN 2024-12-09T12:16:00,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN 2024-12-09T12:16:00,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba4a26e129719340e5696ac56c6b61ac, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:00,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8bc29ffded7f1b74a1ee2986c5e611f8, server=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:00,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a258d2ecd9a494e8d56406b774da28ef, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:00,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:00,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure ebee1500f62faadfac8720740a1e527e, server=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:00,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43f38f304815b66d00adc6292e217f1a, server=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:00,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ceebab6d8b44ccd74ad65e871405953, server=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:00,979 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:00,979 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:00,979 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:00,980 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:00,980 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:00,980 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:00,980 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:00,980 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:00,980 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:00,980 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:00,980 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:00,980 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:00,980 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:00,980 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:00,980 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN 2024-12-09T12:16:00,980 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:00,980 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN 2024-12-09T12:16:00,980 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:00,980 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN 2024-12-09T12:16:00,980 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:00,980 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN 2024-12-09T12:16:00,980 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:00,980 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN 2024-12-09T12:16:00,980 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba4a26e129719340e5696ac56c6b61ac, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:00,981 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8bc29ffded7f1b74a1ee2986c5e611f8, server=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:00,981 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a258d2ecd9a494e8d56406b774da28ef, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:00,981 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:00,981 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure ebee1500f62faadfac8720740a1e527e, server=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:00,981 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43f38f304815b66d00adc6292e217f1a, server=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:00,981 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ceebab6d8b44ccd74ad65e871405953, server=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:00,981 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-12-09T12:16:00,983 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=65 2024-12-09T12:16:00,983 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f15dd41c88ad,38111,1733746548467, table=hbase:meta, region=1588230740 2024-12-09T12:16:00,984 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f15dd41c88ad,38111,1733746548467, state=OPEN 2024-12-09T12:16:00,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:00,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:00,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:00,986 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:00,986 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:00,986 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:00,987 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:00,987 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN to state=OFFLINE, location=null, table=test, region=62a60a0dff0c026a110e755e6c249001 to restore RIT 2024-12-09T12:16:00,988 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN to state=OFFLINE, location=null, table=test, region=ba4a26e129719340e5696ac56c6b61ac to restore RIT 2024-12-09T12:16:00,988 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN to state=OFFLINE, location=null, table=test, region=ebee1500f62faadfac8720740a1e527e to restore RIT 2024-12-09T12:16:00,988 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN to state=OFFLINE, location=null, table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8 to restore RIT 2024-12-09T12:16:00,988 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN to state=OFFLINE, location=null, table=test, region=c8f8bc12489e5c0370ba59c751783a23 to restore RIT 2024-12-09T12:16:00,988 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN to state=OFFLINE, location=null, table=test, region=a258d2ecd9a494e8d56406b774da28ef to restore RIT 2024-12-09T12:16:00,989 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN to state=OFFLINE, location=null, table=test, region=8ceebab6d8b44ccd74ad65e871405953 to restore RIT 2024-12-09T12:16:00,989 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN to state=OFFLINE, location=null, table=test, region=43f38f304815b66d00adc6292e217f1a to restore RIT 2024-12-09T12:16:00,989 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733746590989 2024-12-09T12:16:00,989 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T12:16:00,991 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:00,992 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-09T12:16:00,993 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-09T12:16:00,993 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43f38f304815b66d00adc6292e217f1a, server=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:00,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:00,994 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43f38f304815b66d00adc6292e217f1a, server=f15dd41c88ad,36579,1733746548727 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc7aaad28378d11daf90725c0f9d39f5, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ac6650f2a35b2c131e93ead260da106, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8bc29ffded7f1b74a1ee2986c5e611f8, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ce28e8d0ce113d2637ab328f5ffd3be, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5251eee3971dde9f0f276cfd8632c21, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=5 2024-12-09T12:16:01,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b5306e854c5aec62eff62a18cec5859, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c910d27fbf3265165c7fd59609e57c82, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a258d2ecd9a494e8d56406b774da28ef, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0cf25c9917186a9e3dbd87e44f33615, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9e23742363329565223433d05d86296, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9702c569de70ef36018a26943c3226f, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebee1500f62faadfac8720740a1e527e, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62a60a0dff0c026a110e755e6c249001, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdcb3d892a0c9cf641eaedbbd9ec5937, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a1205a417c23329b514f783f4727022, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43f38f304815b66d00adc6292e217f1a, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab7cda743d5cc81945ffb7c431d6346, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,004 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0adbf1ba01f94cad79f2814d407182f, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,004 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603cf0cbb0e068a5f7727e4abab3a5d5, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,004 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba4a26e129719340e5696ac56c6b61ac, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,004 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0351c7e96a16dae18ff5a8d1664b2b1, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,004 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23280f1743db631eaa35bf3487e69147, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,004 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ceebab6d8b44ccd74ad65e871405953, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,004 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a763ac99f0e3a37f94a31b5f0479530, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,005 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eaca6c428d8bff0cd952b3c2dfe5408, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,005 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8f8bc12489e5c0370ba59c751783a23, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,005 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-09T12:16:01,005 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733746621005 2024-12-09T12:16:01,005 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733746681005 2024-12-09T12:16:01,005 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 15 msec 2024-12-09T12:16:01,005 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-09T12:16:01,005 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,005 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T12:16:01,005 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T12:16:01,005 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-09T12:16:01,005 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-09T12:16:01,005 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-09T12:16:01,006 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T12:16:01,006 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-09T12:16:01,007 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,010 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-09T12:16:01,010 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,010 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,010 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,010 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,010 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,010 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,010 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,010 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,010 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,010 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,010 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,010 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,010 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,010 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,010 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,010 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,010 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,010 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,010 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,010 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,011 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T12:16:01,011 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-09T12:16:01,012 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, REOPEN/MOVE 2024-12-09T12:16:01,014 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true 2024-12-09T12:16:01,014 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,014 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,014 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T12:16:01,014 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,015 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,015 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,015 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:01,015 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:01,015 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:01,015 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN 2024-12-09T12:16:01,015 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN 2024-12-09T12:16:01,015 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN 2024-12-09T12:16:01,015 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN 2024-12-09T12:16:01,015 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN 2024-12-09T12:16:01,015 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba4a26e129719340e5696ac56c6b61ac, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,015 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8bc29ffded7f1b74a1ee2986c5e611f8, server=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,015 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a258d2ecd9a494e8d56406b774da28ef, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,015 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,015 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure ebee1500f62faadfac8720740a1e527e, server=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,015 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43f38f304815b66d00adc6292e217f1a, server=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,015 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,015 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,015 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,015 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,015 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,015 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,015 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,015 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,015 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:01,015 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,015 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:01,016 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,016 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:01,016 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,016 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN 2024-12-09T12:16:01,016 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,016 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN 2024-12-09T12:16:01,016 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,016 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN 2024-12-09T12:16:01,016 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,016 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN 2024-12-09T12:16:01,016 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,016 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN 2024-12-09T12:16:01,016 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba4a26e129719340e5696ac56c6b61ac, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,016 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8bc29ffded7f1b74a1ee2986c5e611f8, server=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,016 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a258d2ecd9a494e8d56406b774da28ef, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,016 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,016 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure ebee1500f62faadfac8720740a1e527e, server=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,016 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43f38f304815b66d00adc6292e217f1a, server=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,016 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-12-09T12:16:01,017 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=65 2024-12-09T12:16:01,017 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f15dd41c88ad,38111,1733746548467, table=hbase:meta, region=1588230740 2024-12-09T12:16:01,017 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f15dd41c88ad,38111,1733746548467, state=OPEN 2024-12-09T12:16:01,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,021 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,021 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,022 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN to state=OFFLINE, location=null, table=test, region=62a60a0dff0c026a110e755e6c249001 to restore RIT 2024-12-09T12:16:01,022 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN to state=OFFLINE, location=null, table=test, region=ba4a26e129719340e5696ac56c6b61ac to restore RIT 2024-12-09T12:16:01,022 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN to state=OFFLINE, location=null, table=test, region=ebee1500f62faadfac8720740a1e527e to restore RIT 2024-12-09T12:16:01,022 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN to state=OFFLINE, location=null, table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8 to restore RIT 2024-12-09T12:16:01,022 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN to state=OFFLINE, location=null, table=test, region=c8f8bc12489e5c0370ba59c751783a23 to restore RIT 2024-12-09T12:16:01,022 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN to state=OFFLINE, location=null, table=test, region=a258d2ecd9a494e8d56406b774da28ef to restore RIT 2024-12-09T12:16:01,022 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN to state=OFFLINE, location=null, table=test, region=8ceebab6d8b44ccd74ad65e871405953 to restore RIT 2024-12-09T12:16:01,022 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN to state=OFFLINE, location=null, table=test, region=43f38f304815b66d00adc6292e217f1a to restore RIT 2024-12-09T12:16:01,022 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,026 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733746591026 2024-12-09T12:16:01,026 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T12:16:01,026 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,028 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-09T12:16:01,034 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-09T12:16:01,035 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure ebee1500f62faadfac8720740a1e527e, server=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,035 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure ebee1500f62faadfac8720740a1e527e, server=f15dd41c88ad,36579,1733746548727 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,039 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc7aaad28378d11daf90725c0f9d39f5, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,040 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ac6650f2a35b2c131e93ead260da106, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,040 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8bc29ffded7f1b74a1ee2986c5e611f8, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,040 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ce28e8d0ce113d2637ab328f5ffd3be, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,041 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5251eee3971dde9f0f276cfd8632c21, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=5 2024-12-09T12:16:01,041 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b5306e854c5aec62eff62a18cec5859, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,041 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c910d27fbf3265165c7fd59609e57c82, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,041 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a258d2ecd9a494e8d56406b774da28ef, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,041 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0cf25c9917186a9e3dbd87e44f33615, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9e23742363329565223433d05d86296, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9702c569de70ef36018a26943c3226f, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebee1500f62faadfac8720740a1e527e, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62a60a0dff0c026a110e755e6c249001, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdcb3d892a0c9cf641eaedbbd9ec5937, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a1205a417c23329b514f783f4727022, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43f38f304815b66d00adc6292e217f1a, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab7cda743d5cc81945ffb7c431d6346, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0adbf1ba01f94cad79f2814d407182f, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603cf0cbb0e068a5f7727e4abab3a5d5, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba4a26e129719340e5696ac56c6b61ac, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0351c7e96a16dae18ff5a8d1664b2b1, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23280f1743db631eaa35bf3487e69147, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ceebab6d8b44ccd74ad65e871405953, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a763ac99f0e3a37f94a31b5f0479530, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eaca6c428d8bff0cd952b3c2dfe5408, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8f8bc12489e5c0370ba59c751783a23, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,047 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-09T12:16:01,047 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733746621047 2024-12-09T12:16:01,047 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733746681047 2024-12-09T12:16:01,047 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 20 msec 2024-12-09T12:16:01,047 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-09T12:16:01,047 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,047 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T12:16:01,047 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T12:16:01,048 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-09T12:16:01,048 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-09T12:16:01,048 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-09T12:16:01,048 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T12:16:01,049 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-09T12:16:01,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-09T12:16:01,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T12:16:01,056 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-09T12:16:01,056 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, REOPEN/MOVE 2024-12-09T12:16:01,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true 2024-12-09T12:16:01,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T12:16:01,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:01,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:01,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:01,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN 2024-12-09T12:16:01,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN 2024-12-09T12:16:01,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN 2024-12-09T12:16:01,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN 2024-12-09T12:16:01,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN 2024-12-09T12:16:01,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba4a26e129719340e5696ac56c6b61ac, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8bc29ffded7f1b74a1ee2986c5e611f8, server=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a258d2ecd9a494e8d56406b774da28ef, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure ebee1500f62faadfac8720740a1e527e, server=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,060 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,060 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,060 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,060 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,060 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,060 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,060 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,060 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,060 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:01,060 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,060 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:01,060 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,060 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:01,060 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,060 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN 2024-12-09T12:16:01,060 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,060 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN 2024-12-09T12:16:01,060 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,060 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN 2024-12-09T12:16:01,060 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,060 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN 2024-12-09T12:16:01,061 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,061 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN 2024-12-09T12:16:01,061 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba4a26e129719340e5696ac56c6b61ac, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,061 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8bc29ffded7f1b74a1ee2986c5e611f8, server=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,061 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a258d2ecd9a494e8d56406b774da28ef, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,061 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,061 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure ebee1500f62faadfac8720740a1e527e, server=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,061 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-12-09T12:16:01,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=65 2024-12-09T12:16:01,063 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f15dd41c88ad,38111,1733746548467, table=hbase:meta, region=1588230740 2024-12-09T12:16:01,063 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f15dd41c88ad,38111,1733746548467, state=OPEN 2024-12-09T12:16:01,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,066 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,066 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,066 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,067 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN to state=OFFLINE, location=null, table=test, region=62a60a0dff0c026a110e755e6c249001 to restore RIT 2024-12-09T12:16:01,068 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN to state=OFFLINE, location=null, table=test, region=ba4a26e129719340e5696ac56c6b61ac to restore RIT 2024-12-09T12:16:01,068 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN to state=OFFLINE, location=null, table=test, region=ebee1500f62faadfac8720740a1e527e to restore RIT 2024-12-09T12:16:01,068 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN to state=OFFLINE, location=null, table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8 to restore RIT 2024-12-09T12:16:01,068 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN to state=OFFLINE, location=null, table=test, region=c8f8bc12489e5c0370ba59c751783a23 to restore RIT 2024-12-09T12:16:01,068 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN to state=OFFLINE, location=null, table=test, region=a258d2ecd9a494e8d56406b774da28ef to restore RIT 2024-12-09T12:16:01,068 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN to state=OFFLINE, location=null, table=test, region=8ceebab6d8b44ccd74ad65e871405953 to restore RIT 2024-12-09T12:16:01,068 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN to state=OFFLINE, location=null, table=test, region=43f38f304815b66d00adc6292e217f1a to restore RIT 2024-12-09T12:16:01,069 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733746591069 2024-12-09T12:16:01,069 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T12:16:01,069 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,070 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-09T12:16:01,075 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-09T12:16:01,075 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,075 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,079 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc7aaad28378d11daf90725c0f9d39f5, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,079 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ac6650f2a35b2c131e93ead260da106, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,079 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8bc29ffded7f1b74a1ee2986c5e611f8, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,079 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ce28e8d0ce113d2637ab328f5ffd3be, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5251eee3971dde9f0f276cfd8632c21, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=5 2024-12-09T12:16:01,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b5306e854c5aec62eff62a18cec5859, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c910d27fbf3265165c7fd59609e57c82, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a258d2ecd9a494e8d56406b774da28ef, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0cf25c9917186a9e3dbd87e44f33615, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9e23742363329565223433d05d86296, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9702c569de70ef36018a26943c3226f, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebee1500f62faadfac8720740a1e527e, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62a60a0dff0c026a110e755e6c249001, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdcb3d892a0c9cf641eaedbbd9ec5937, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a1205a417c23329b514f783f4727022, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43f38f304815b66d00adc6292e217f1a, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab7cda743d5cc81945ffb7c431d6346, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0adbf1ba01f94cad79f2814d407182f, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603cf0cbb0e068a5f7727e4abab3a5d5, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba4a26e129719340e5696ac56c6b61ac, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0351c7e96a16dae18ff5a8d1664b2b1, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23280f1743db631eaa35bf3487e69147, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ceebab6d8b44ccd74ad65e871405953, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a763ac99f0e3a37f94a31b5f0479530, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eaca6c428d8bff0cd952b3c2dfe5408, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8f8bc12489e5c0370ba59c751783a23, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,084 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-09T12:16:01,084 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733746621084 2024-12-09T12:16:01,084 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733746681084 2024-12-09T12:16:01,084 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 15 msec 2024-12-09T12:16:01,084 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-09T12:16:01,084 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,084 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T12:16:01,084 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T12:16:01,085 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-09T12:16:01,085 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-09T12:16:01,085 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-09T12:16:01,085 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T12:16:01,087 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-12-09T12:16:01,090 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-09T12:16:01,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T12:16:01,092 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-09T12:16:01,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, REOPEN/MOVE 2024-12-09T12:16:01,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true 2024-12-09T12:16:01,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T12:16:01,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:01,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:01,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:01,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN 2024-12-09T12:16:01,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN 2024-12-09T12:16:01,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN 2024-12-09T12:16:01,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN 2024-12-09T12:16:01,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN 2024-12-09T12:16:01,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba4a26e129719340e5696ac56c6b61ac, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8bc29ffded7f1b74a1ee2986c5e611f8, server=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a258d2ecd9a494e8d56406b774da28ef, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,095 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,096 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,096 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,096 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,096 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,096 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,096 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,096 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,096 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:01,096 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,096 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:01,096 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,096 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:01,096 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,096 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN 2024-12-09T12:16:01,096 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,096 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN 2024-12-09T12:16:01,096 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,096 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN 2024-12-09T12:16:01,096 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,096 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN 2024-12-09T12:16:01,097 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,097 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN 2024-12-09T12:16:01,097 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba4a26e129719340e5696ac56c6b61ac, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,097 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8bc29ffded7f1b74a1ee2986c5e611f8, server=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,097 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a258d2ecd9a494e8d56406b774da28ef, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,097 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,097 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-12-09T12:16:01,100 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=65 2024-12-09T12:16:01,100 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f15dd41c88ad,38111,1733746548467, table=hbase:meta, region=1588230740 2024-12-09T12:16:01,101 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f15dd41c88ad,38111,1733746548467, state=OPEN 2024-12-09T12:16:01,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,104 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,104 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN to state=OFFLINE, location=null, table=test, region=62a60a0dff0c026a110e755e6c249001 to restore RIT 2024-12-09T12:16:01,105 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN to state=OFFLINE, location=null, table=test, region=ba4a26e129719340e5696ac56c6b61ac to restore RIT 2024-12-09T12:16:01,105 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN to state=OFFLINE, location=null, table=test, region=ebee1500f62faadfac8720740a1e527e to restore RIT 2024-12-09T12:16:01,105 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN to state=OFFLINE, location=null, table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8 to restore RIT 2024-12-09T12:16:01,105 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN to state=OFFLINE, location=null, table=test, region=c8f8bc12489e5c0370ba59c751783a23 to restore RIT 2024-12-09T12:16:01,105 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN to state=OFFLINE, location=null, table=test, region=a258d2ecd9a494e8d56406b774da28ef to restore RIT 2024-12-09T12:16:01,105 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN to state=OFFLINE, location=null, table=test, region=8ceebab6d8b44ccd74ad65e871405953 to restore RIT 2024-12-09T12:16:01,105 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN to state=OFFLINE, location=null, table=test, region=43f38f304815b66d00adc6292e217f1a to restore RIT 2024-12-09T12:16:01,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,106 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,107 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,108 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733746591108 2024-12-09T12:16:01,108 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T12:16:01,108 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,109 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-09T12:16:01,116 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-09T12:16:01,116 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a258d2ecd9a494e8d56406b774da28ef, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,116 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a258d2ecd9a494e8d56406b774da28ef, server=f15dd41c88ad,38111,1733746548467 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc7aaad28378d11daf90725c0f9d39f5, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ac6650f2a35b2c131e93ead260da106, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8bc29ffded7f1b74a1ee2986c5e611f8, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ce28e8d0ce113d2637ab328f5ffd3be, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5251eee3971dde9f0f276cfd8632c21, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=5 2024-12-09T12:16:01,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b5306e854c5aec62eff62a18cec5859, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c910d27fbf3265165c7fd59609e57c82, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a258d2ecd9a494e8d56406b774da28ef, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0cf25c9917186a9e3dbd87e44f33615, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9e23742363329565223433d05d86296, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9702c569de70ef36018a26943c3226f, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebee1500f62faadfac8720740a1e527e, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62a60a0dff0c026a110e755e6c249001, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdcb3d892a0c9cf641eaedbbd9ec5937, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a1205a417c23329b514f783f4727022, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,122 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43f38f304815b66d00adc6292e217f1a, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,122 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab7cda743d5cc81945ffb7c431d6346, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,122 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0adbf1ba01f94cad79f2814d407182f, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,122 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603cf0cbb0e068a5f7727e4abab3a5d5, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,123 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba4a26e129719340e5696ac56c6b61ac, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,123 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0351c7e96a16dae18ff5a8d1664b2b1, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,123 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23280f1743db631eaa35bf3487e69147, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,123 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ceebab6d8b44ccd74ad65e871405953, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,123 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a763ac99f0e3a37f94a31b5f0479530, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,124 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eaca6c428d8bff0cd952b3c2dfe5408, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,124 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8f8bc12489e5c0370ba59c751783a23, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,124 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-09T12:16:01,125 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733746621125 2024-12-09T12:16:01,125 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733746681125 2024-12-09T12:16:01,125 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 16 msec 2024-12-09T12:16:01,125 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-09T12:16:01,125 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,125 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T12:16:01,125 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T12:16:01,128 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-09T12:16:01,128 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-09T12:16:01,128 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-09T12:16:01,128 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T12:16:01,129 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-09T12:16:01,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-09T12:16:01,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,133 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T12:16:01,133 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-09T12:16:01,134 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, REOPEN/MOVE 2024-12-09T12:16:01,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true 2024-12-09T12:16:01,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T12:16:01,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:01,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:01,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:01,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN 2024-12-09T12:16:01,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN 2024-12-09T12:16:01,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN 2024-12-09T12:16:01,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN 2024-12-09T12:16:01,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN 2024-12-09T12:16:01,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba4a26e129719340e5696ac56c6b61ac, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8bc29ffded7f1b74a1ee2986c5e611f8, server=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a258d2ecd9a494e8d56406b774da28ef, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,137 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,137 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,137 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,137 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,137 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,137 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,137 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,137 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,137 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:01,137 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,137 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:01,137 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,137 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:01,137 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,137 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN 2024-12-09T12:16:01,137 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,137 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN 2024-12-09T12:16:01,137 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,137 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN 2024-12-09T12:16:01,137 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,137 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN 2024-12-09T12:16:01,137 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,137 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN 2024-12-09T12:16:01,138 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba4a26e129719340e5696ac56c6b61ac, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,138 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8bc29ffded7f1b74a1ee2986c5e611f8, server=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,138 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a258d2ecd9a494e8d56406b774da28ef, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,138 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-12-09T12:16:01,141 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=65 2024-12-09T12:16:01,141 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f15dd41c88ad,38111,1733746548467, table=hbase:meta, region=1588230740 2024-12-09T12:16:01,141 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f15dd41c88ad,38111,1733746548467, state=OPEN 2024-12-09T12:16:01,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,143 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,144 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN to state=OFFLINE, location=null, table=test, region=62a60a0dff0c026a110e755e6c249001 to restore RIT 2024-12-09T12:16:01,144 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN to state=OFFLINE, location=null, table=test, region=ba4a26e129719340e5696ac56c6b61ac to restore RIT 2024-12-09T12:16:01,144 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN to state=OFFLINE, location=null, table=test, region=ebee1500f62faadfac8720740a1e527e to restore RIT 2024-12-09T12:16:01,144 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN to state=OFFLINE, location=null, table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8 to restore RIT 2024-12-09T12:16:01,144 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN to state=OFFLINE, location=null, table=test, region=c8f8bc12489e5c0370ba59c751783a23 to restore RIT 2024-12-09T12:16:01,144 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN to state=OFFLINE, location=null, table=test, region=a258d2ecd9a494e8d56406b774da28ef to restore RIT 2024-12-09T12:16:01,144 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN to state=OFFLINE, location=null, table=test, region=8ceebab6d8b44ccd74ad65e871405953 to restore RIT 2024-12-09T12:16:01,145 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN to state=OFFLINE, location=null, table=test, region=43f38f304815b66d00adc6292e217f1a to restore RIT 2024-12-09T12:16:01,146 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733746591145 2024-12-09T12:16:01,146 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T12:16:01,146 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,146 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,147 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-09T12:16:01,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,149 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,152 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-09T12:16:01,152 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8bc29ffded7f1b74a1ee2986c5e611f8, server=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,153 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc7aaad28378d11daf90725c0f9d39f5, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,152 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8bc29ffded7f1b74a1ee2986c5e611f8, server=f15dd41c88ad,36579,1733746548727 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,153 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ac6650f2a35b2c131e93ead260da106, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,153 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8bc29ffded7f1b74a1ee2986c5e611f8, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,153 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ce28e8d0ce113d2637ab328f5ffd3be, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5251eee3971dde9f0f276cfd8632c21, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=5 2024-12-09T12:16:01,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b5306e854c5aec62eff62a18cec5859, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c910d27fbf3265165c7fd59609e57c82, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a258d2ecd9a494e8d56406b774da28ef, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0cf25c9917186a9e3dbd87e44f33615, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9e23742363329565223433d05d86296, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9702c569de70ef36018a26943c3226f, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebee1500f62faadfac8720740a1e527e, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62a60a0dff0c026a110e755e6c249001, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdcb3d892a0c9cf641eaedbbd9ec5937, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a1205a417c23329b514f783f4727022, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43f38f304815b66d00adc6292e217f1a, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab7cda743d5cc81945ffb7c431d6346, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0adbf1ba01f94cad79f2814d407182f, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603cf0cbb0e068a5f7727e4abab3a5d5, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba4a26e129719340e5696ac56c6b61ac, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0351c7e96a16dae18ff5a8d1664b2b1, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23280f1743db631eaa35bf3487e69147, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ceebab6d8b44ccd74ad65e871405953, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a763ac99f0e3a37f94a31b5f0479530, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eaca6c428d8bff0cd952b3c2dfe5408, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8f8bc12489e5c0370ba59c751783a23, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,157 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-09T12:16:01,157 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733746621157 2024-12-09T12:16:01,157 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733746681157 2024-12-09T12:16:01,157 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-12-09T12:16:01,158 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-09T12:16:01,158 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,158 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T12:16:01,158 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T12:16:01,158 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-09T12:16:01,158 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-09T12:16:01,158 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-09T12:16:01,158 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T12:16:01,159 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-09T12:16:01,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-09T12:16:01,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T12:16:01,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-09T12:16:01,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, REOPEN/MOVE 2024-12-09T12:16:01,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true 2024-12-09T12:16:01,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T12:16:01,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:01,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:01,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:01,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN 2024-12-09T12:16:01,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN 2024-12-09T12:16:01,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN 2024-12-09T12:16:01,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN 2024-12-09T12:16:01,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN 2024-12-09T12:16:01,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba4a26e129719340e5696ac56c6b61ac, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8bc29ffded7f1b74a1ee2986c5e611f8, server=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,167 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,167 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,167 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,167 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,167 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,167 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,167 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,167 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,167 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:01,167 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,167 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:01,167 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,167 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:01,167 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,168 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN 2024-12-09T12:16:01,168 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,168 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN 2024-12-09T12:16:01,168 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,168 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN 2024-12-09T12:16:01,168 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,168 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN 2024-12-09T12:16:01,168 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,168 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN 2024-12-09T12:16:01,168 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba4a26e129719340e5696ac56c6b61ac, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,168 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8bc29ffded7f1b74a1ee2986c5e611f8, server=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,168 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-12-09T12:16:01,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=65 2024-12-09T12:16:01,169 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f15dd41c88ad,38111,1733746548467, table=hbase:meta, region=1588230740 2024-12-09T12:16:01,169 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f15dd41c88ad,38111,1733746548467, state=OPEN 2024-12-09T12:16:01,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,173 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,174 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,175 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,177 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN to state=OFFLINE, location=null, table=test, region=62a60a0dff0c026a110e755e6c249001 to restore RIT 2024-12-09T12:16:01,177 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN to state=OFFLINE, location=null, table=test, region=ba4a26e129719340e5696ac56c6b61ac to restore RIT 2024-12-09T12:16:01,177 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN to state=OFFLINE, location=null, table=test, region=ebee1500f62faadfac8720740a1e527e to restore RIT 2024-12-09T12:16:01,177 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN to state=OFFLINE, location=null, table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8 to restore RIT 2024-12-09T12:16:01,177 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN to state=OFFLINE, location=null, table=test, region=c8f8bc12489e5c0370ba59c751783a23 to restore RIT 2024-12-09T12:16:01,177 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN to state=OFFLINE, location=null, table=test, region=a258d2ecd9a494e8d56406b774da28ef to restore RIT 2024-12-09T12:16:01,177 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN to state=OFFLINE, location=null, table=test, region=8ceebab6d8b44ccd74ad65e871405953 to restore RIT 2024-12-09T12:16:01,178 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN to state=OFFLINE, location=null, table=test, region=43f38f304815b66d00adc6292e217f1a to restore RIT 2024-12-09T12:16:01,188 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733746591188 2024-12-09T12:16:01,188 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T12:16:01,188 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,190 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-09T12:16:01,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc7aaad28378d11daf90725c0f9d39f5, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,200 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ac6650f2a35b2c131e93ead260da106, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,200 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8bc29ffded7f1b74a1ee2986c5e611f8, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,200 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ce28e8d0ce113d2637ab328f5ffd3be, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,200 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5251eee3971dde9f0f276cfd8632c21, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=5 2024-12-09T12:16:01,200 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b5306e854c5aec62eff62a18cec5859, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,200 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c910d27fbf3265165c7fd59609e57c82, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,201 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a258d2ecd9a494e8d56406b774da28ef, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,201 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0cf25c9917186a9e3dbd87e44f33615, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,201 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9e23742363329565223433d05d86296, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,201 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9702c569de70ef36018a26943c3226f, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,201 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebee1500f62faadfac8720740a1e527e, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,201 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62a60a0dff0c026a110e755e6c249001, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdcb3d892a0c9cf641eaedbbd9ec5937, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a1205a417c23329b514f783f4727022, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43f38f304815b66d00adc6292e217f1a, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab7cda743d5cc81945ffb7c431d6346, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0adbf1ba01f94cad79f2814d407182f, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,203 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603cf0cbb0e068a5f7727e4abab3a5d5, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,203 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-09T12:16:01,203 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba4a26e129719340e5696ac56c6b61ac, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,203 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba4a26e129719340e5696ac56c6b61ac, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,203 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba4a26e129719340e5696ac56c6b61ac, server=f15dd41c88ad,38111,1733746548467 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,203 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0351c7e96a16dae18ff5a8d1664b2b1, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,203 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23280f1743db631eaa35bf3487e69147, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,204 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ceebab6d8b44ccd74ad65e871405953, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,204 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a763ac99f0e3a37f94a31b5f0479530, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,204 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eaca6c428d8bff0cd952b3c2dfe5408, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,204 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8f8bc12489e5c0370ba59c751783a23, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,204 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-09T12:16:01,204 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733746621204 2024-12-09T12:16:01,204 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733746681204 2024-12-09T12:16:01,204 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 16 msec 2024-12-09T12:16:01,204 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-09T12:16:01,204 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,204 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T12:16:01,205 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T12:16:01,207 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-09T12:16:01,207 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-09T12:16:01,207 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-09T12:16:01,207 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T12:16:01,208 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-09T12:16:01,210 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-09T12:16:01,211 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,211 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,211 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,211 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,211 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,211 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,211 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,211 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,211 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,211 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,211 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,211 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,211 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,211 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,211 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,212 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T12:16:01,213 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-09T12:16:01,213 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, REOPEN/MOVE 2024-12-09T12:16:01,215 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true 2024-12-09T12:16:01,215 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,215 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,215 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T12:16:01,215 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,215 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,215 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,215 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:01,215 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:01,215 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:01,215 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN 2024-12-09T12:16:01,215 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN 2024-12-09T12:16:01,215 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN 2024-12-09T12:16:01,215 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN 2024-12-09T12:16:01,215 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN 2024-12-09T12:16:01,215 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba4a26e129719340e5696ac56c6b61ac, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,215 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,215 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,215 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,215 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,215 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,215 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,215 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,215 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,216 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:01,216 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,216 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:01,216 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,216 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:01,216 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,216 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN 2024-12-09T12:16:01,216 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,216 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN 2024-12-09T12:16:01,216 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,216 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN 2024-12-09T12:16:01,216 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,216 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN 2024-12-09T12:16:01,216 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,216 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,216 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN 2024-12-09T12:16:01,216 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba4a26e129719340e5696ac56c6b61ac, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,216 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-12-09T12:16:01,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=65 2024-12-09T12:16:01,217 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f15dd41c88ad,38111,1733746548467, table=hbase:meta, region=1588230740 2024-12-09T12:16:01,217 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f15dd41c88ad,38111,1733746548467, state=OPEN 2024-12-09T12:16:01,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,219 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,219 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,220 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,220 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN to state=OFFLINE, location=null, table=test, region=62a60a0dff0c026a110e755e6c249001 to restore RIT 2024-12-09T12:16:01,221 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN to state=OFFLINE, location=null, table=test, region=ba4a26e129719340e5696ac56c6b61ac to restore RIT 2024-12-09T12:16:01,221 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN to state=OFFLINE, location=null, table=test, region=ebee1500f62faadfac8720740a1e527e to restore RIT 2024-12-09T12:16:01,221 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN to state=OFFLINE, location=null, table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8 to restore RIT 2024-12-09T12:16:01,221 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN to state=OFFLINE, location=null, table=test, region=c8f8bc12489e5c0370ba59c751783a23 to restore RIT 2024-12-09T12:16:01,221 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN to state=OFFLINE, location=null, table=test, region=a258d2ecd9a494e8d56406b774da28ef to restore RIT 2024-12-09T12:16:01,221 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN to state=OFFLINE, location=null, table=test, region=8ceebab6d8b44ccd74ad65e871405953 to restore RIT 2024-12-09T12:16:01,221 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN to state=OFFLINE, location=null, table=test, region=43f38f304815b66d00adc6292e217f1a to restore RIT 2024-12-09T12:16:01,225 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733746591225 2024-12-09T12:16:01,225 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T12:16:01,226 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,227 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-09T12:16:01,233 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-09T12:16:01,233 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN 2024-12-09T12:16:01,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,233 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,233 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc7aaad28378d11daf90725c0f9d39f5, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ac6650f2a35b2c131e93ead260da106, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8bc29ffded7f1b74a1ee2986c5e611f8, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ce28e8d0ce113d2637ab328f5ffd3be, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5251eee3971dde9f0f276cfd8632c21, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=5 2024-12-09T12:16:01,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b5306e854c5aec62eff62a18cec5859, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c910d27fbf3265165c7fd59609e57c82, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a258d2ecd9a494e8d56406b774da28ef, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0cf25c9917186a9e3dbd87e44f33615, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9e23742363329565223433d05d86296, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9702c569de70ef36018a26943c3226f, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebee1500f62faadfac8720740a1e527e, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62a60a0dff0c026a110e755e6c249001, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdcb3d892a0c9cf641eaedbbd9ec5937, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a1205a417c23329b514f783f4727022, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43f38f304815b66d00adc6292e217f1a, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab7cda743d5cc81945ffb7c431d6346, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0adbf1ba01f94cad79f2814d407182f, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603cf0cbb0e068a5f7727e4abab3a5d5, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba4a26e129719340e5696ac56c6b61ac, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0351c7e96a16dae18ff5a8d1664b2b1, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23280f1743db631eaa35bf3487e69147, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ceebab6d8b44ccd74ad65e871405953, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a763ac99f0e3a37f94a31b5f0479530, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eaca6c428d8bff0cd952b3c2dfe5408, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8f8bc12489e5c0370ba59c751783a23, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,238 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-09T12:16:01,238 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733746621238 2024-12-09T12:16:01,238 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733746681238 2024-12-09T12:16:01,238 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-12-09T12:16:01,238 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-09T12:16:01,239 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,239 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T12:16:01,239 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T12:16:01,242 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-09T12:16:01,243 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-09T12:16:01,243 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-09T12:16:01,243 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T12:16:01,247 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 3 msec 2024-12-09T12:16:01,249 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-09T12:16:01,249 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,249 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,249 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,249 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,249 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,249 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,249 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,249 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,249 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,253 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,253 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,253 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,253 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,254 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T12:16:01,255 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-09T12:16:01,255 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, REOPEN/MOVE 2024-12-09T12:16:01,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true 2024-12-09T12:16:01,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T12:16:01,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:01,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:01,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:01,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN 2024-12-09T12:16:01,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN 2024-12-09T12:16:01,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN 2024-12-09T12:16:01,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN 2024-12-09T12:16:01,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN 2024-12-09T12:16:01,257 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,257 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,258 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,258 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:01,258 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,258 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:01,258 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,258 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:01,258 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,258 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN 2024-12-09T12:16:01,258 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,258 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN 2024-12-09T12:16:01,258 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,258 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN 2024-12-09T12:16:01,258 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,258 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN 2024-12-09T12:16:01,258 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,258 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN 2024-12-09T12:16:01,259 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-12-09T12:16:01,263 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=65 2024-12-09T12:16:01,263 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f15dd41c88ad,38111,1733746548467, table=hbase:meta, region=1588230740 2024-12-09T12:16:01,263 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f15dd41c88ad,38111,1733746548467, state=OPEN 2024-12-09T12:16:01,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,265 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,266 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,267 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,268 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN to state=OFFLINE, location=null, table=test, region=62a60a0dff0c026a110e755e6c249001 to restore RIT 2024-12-09T12:16:01,268 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN to state=OFFLINE, location=null, table=test, region=ba4a26e129719340e5696ac56c6b61ac to restore RIT 2024-12-09T12:16:01,268 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN to state=OFFLINE, location=null, table=test, region=ebee1500f62faadfac8720740a1e527e to restore RIT 2024-12-09T12:16:01,269 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN to state=OFFLINE, location=null, table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8 to restore RIT 2024-12-09T12:16:01,269 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN to state=OFFLINE, location=null, table=test, region=c8f8bc12489e5c0370ba59c751783a23 to restore RIT 2024-12-09T12:16:01,269 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN to state=OFFLINE, location=null, table=test, region=a258d2ecd9a494e8d56406b774da28ef to restore RIT 2024-12-09T12:16:01,269 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN to state=OFFLINE, location=null, table=test, region=8ceebab6d8b44ccd74ad65e871405953 to restore RIT 2024-12-09T12:16:01,269 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=43f38f304815b66d00adc6292e217f1a, ASSIGN to state=OFFLINE, location=null, table=test, region=43f38f304815b66d00adc6292e217f1a to restore RIT 2024-12-09T12:16:01,270 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733746591270 2024-12-09T12:16:01,270 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T12:16:01,270 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,272 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-09T12:16:01,277 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-09T12:16:01,277 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN 2024-12-09T12:16:01,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,278 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc7aaad28378d11daf90725c0f9d39f5, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ac6650f2a35b2c131e93ead260da106, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8bc29ffded7f1b74a1ee2986c5e611f8, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ce28e8d0ce113d2637ab328f5ffd3be, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5251eee3971dde9f0f276cfd8632c21, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=5 2024-12-09T12:16:01,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b5306e854c5aec62eff62a18cec5859, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c910d27fbf3265165c7fd59609e57c82, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a258d2ecd9a494e8d56406b774da28ef, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0cf25c9917186a9e3dbd87e44f33615, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9e23742363329565223433d05d86296, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9702c569de70ef36018a26943c3226f, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebee1500f62faadfac8720740a1e527e, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62a60a0dff0c026a110e755e6c249001, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdcb3d892a0c9cf641eaedbbd9ec5937, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a1205a417c23329b514f783f4727022, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43f38f304815b66d00adc6292e217f1a, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab7cda743d5cc81945ffb7c431d6346, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0adbf1ba01f94cad79f2814d407182f, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603cf0cbb0e068a5f7727e4abab3a5d5, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba4a26e129719340e5696ac56c6b61ac, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0351c7e96a16dae18ff5a8d1664b2b1, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23280f1743db631eaa35bf3487e69147, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,286 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ceebab6d8b44ccd74ad65e871405953, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,286 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a763ac99f0e3a37f94a31b5f0479530, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,286 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eaca6c428d8bff0cd952b3c2dfe5408, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,286 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8f8bc12489e5c0370ba59c751783a23, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,286 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-09T12:16:01,286 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733746621286 2024-12-09T12:16:01,286 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733746681286 2024-12-09T12:16:01,286 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 15 msec 2024-12-09T12:16:01,286 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-09T12:16:01,286 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,286 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T12:16:01,286 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T12:16:01,287 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-09T12:16:01,287 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-09T12:16:01,287 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-09T12:16:01,287 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T12:16:01,288 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-09T12:16:01,290 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-09T12:16:01,290 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,290 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,290 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,291 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T12:16:01,292 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-09T12:16:01,292 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, REOPEN/MOVE 2024-12-09T12:16:01,293 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true 2024-12-09T12:16:01,293 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,293 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,293 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T12:16:01,293 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,293 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,293 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,293 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:01,293 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:01,294 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:01,294 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN 2024-12-09T12:16:01,294 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN 2024-12-09T12:16:01,294 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN 2024-12-09T12:16:01,294 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN 2024-12-09T12:16:01,294 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,294 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,294 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,294 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,294 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,294 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,294 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,294 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,294 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:01,294 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,294 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:01,294 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,294 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:01,294 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,294 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN 2024-12-09T12:16:01,294 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,294 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN 2024-12-09T12:16:01,294 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,295 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN 2024-12-09T12:16:01,295 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,295 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN 2024-12-09T12:16:01,295 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-09T12:16:01,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=65 2024-12-09T12:16:01,296 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f15dd41c88ad,38111,1733746548467, table=hbase:meta, region=1588230740 2024-12-09T12:16:01,296 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f15dd41c88ad,38111,1733746548467, state=OPEN 2024-12-09T12:16:01,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,299 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,299 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,299 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,299 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN to state=OFFLINE, location=null, table=test, region=62a60a0dff0c026a110e755e6c249001 to restore RIT 2024-12-09T12:16:01,300 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN to state=OFFLINE, location=null, table=test, region=ba4a26e129719340e5696ac56c6b61ac to restore RIT 2024-12-09T12:16:01,300 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN to state=OFFLINE, location=null, table=test, region=ebee1500f62faadfac8720740a1e527e to restore RIT 2024-12-09T12:16:01,300 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN to state=OFFLINE, location=null, table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8 to restore RIT 2024-12-09T12:16:01,300 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN to state=OFFLINE, location=null, table=test, region=c8f8bc12489e5c0370ba59c751783a23 to restore RIT 2024-12-09T12:16:01,300 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN to state=OFFLINE, location=null, table=test, region=a258d2ecd9a494e8d56406b774da28ef to restore RIT 2024-12-09T12:16:01,300 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8ceebab6d8b44ccd74ad65e871405953, ASSIGN to state=OFFLINE, location=null, table=test, region=8ceebab6d8b44ccd74ad65e871405953 to restore RIT 2024-12-09T12:16:01,302 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733746591302 2024-12-09T12:16:01,302 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T12:16:01,304 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,305 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-09T12:16:01,306 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-09T12:16:01,306 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN 2024-12-09T12:16:01,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,306 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,308 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc7aaad28378d11daf90725c0f9d39f5, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ac6650f2a35b2c131e93ead260da106, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8bc29ffded7f1b74a1ee2986c5e611f8, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ce28e8d0ce113d2637ab328f5ffd3be, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5251eee3971dde9f0f276cfd8632c21, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=5 2024-12-09T12:16:01,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b5306e854c5aec62eff62a18cec5859, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c910d27fbf3265165c7fd59609e57c82, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a258d2ecd9a494e8d56406b774da28ef, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0cf25c9917186a9e3dbd87e44f33615, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9e23742363329565223433d05d86296, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9702c569de70ef36018a26943c3226f, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebee1500f62faadfac8720740a1e527e, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62a60a0dff0c026a110e755e6c249001, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdcb3d892a0c9cf641eaedbbd9ec5937, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a1205a417c23329b514f783f4727022, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43f38f304815b66d00adc6292e217f1a, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab7cda743d5cc81945ffb7c431d6346, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0adbf1ba01f94cad79f2814d407182f, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603cf0cbb0e068a5f7727e4abab3a5d5, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba4a26e129719340e5696ac56c6b61ac, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0351c7e96a16dae18ff5a8d1664b2b1, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23280f1743db631eaa35bf3487e69147, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ceebab6d8b44ccd74ad65e871405953, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a763ac99f0e3a37f94a31b5f0479530, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eaca6c428d8bff0cd952b3c2dfe5408, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8f8bc12489e5c0370ba59c751783a23, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,313 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-09T12:16:01,313 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733746621313 2024-12-09T12:16:01,313 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733746681313 2024-12-09T12:16:01,313 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-12-09T12:16:01,313 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-09T12:16:01,313 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,313 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T12:16:01,313 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T12:16:01,314 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-09T12:16:01,314 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-09T12:16:01,314 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-09T12:16:01,314 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T12:16:01,317 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-12-09T12:16:01,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-09T12:16:01,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,320 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T12:16:01,320 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-09T12:16:01,320 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, REOPEN/MOVE 2024-12-09T12:16:01,320 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,328 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true 2024-12-09T12:16:01,329 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,329 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,329 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T12:16:01,329 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,329 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,329 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,329 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:01,329 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:01,329 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:01,329 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN 2024-12-09T12:16:01,329 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN 2024-12-09T12:16:01,329 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN 2024-12-09T12:16:01,329 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,329 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,329 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,329 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,329 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,330 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,330 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,330 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,330 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:01,330 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,330 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:01,330 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,330 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:01,330 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,330 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN 2024-12-09T12:16:01,330 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,330 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN 2024-12-09T12:16:01,330 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,330 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN 2024-12-09T12:16:01,330 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-12-09T12:16:01,332 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=65 2024-12-09T12:16:01,332 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f15dd41c88ad,38111,1733746548467, table=hbase:meta, region=1588230740 2024-12-09T12:16:01,332 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f15dd41c88ad,38111,1733746548467, state=OPEN 2024-12-09T12:16:01,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,341 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,342 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,342 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,347 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN to state=OFFLINE, location=null, table=test, region=62a60a0dff0c026a110e755e6c249001 to restore RIT 2024-12-09T12:16:01,347 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN to state=OFFLINE, location=null, table=test, region=ba4a26e129719340e5696ac56c6b61ac to restore RIT 2024-12-09T12:16:01,347 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN to state=OFFLINE, location=null, table=test, region=ebee1500f62faadfac8720740a1e527e to restore RIT 2024-12-09T12:16:01,347 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN to state=OFFLINE, location=null, table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8 to restore RIT 2024-12-09T12:16:01,347 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN to state=OFFLINE, location=null, table=test, region=c8f8bc12489e5c0370ba59c751783a23 to restore RIT 2024-12-09T12:16:01,347 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a258d2ecd9a494e8d56406b774da28ef, ASSIGN to state=OFFLINE, location=null, table=test, region=a258d2ecd9a494e8d56406b774da28ef to restore RIT 2024-12-09T12:16:01,348 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733746591348 2024-12-09T12:16:01,348 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T12:16:01,348 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,354 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-09T12:16:01,358 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-09T12:16:01,358 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN 2024-12-09T12:16:01,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,358 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc7aaad28378d11daf90725c0f9d39f5, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ac6650f2a35b2c131e93ead260da106, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8bc29ffded7f1b74a1ee2986c5e611f8, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ce28e8d0ce113d2637ab328f5ffd3be, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5251eee3971dde9f0f276cfd8632c21, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=5 2024-12-09T12:16:01,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b5306e854c5aec62eff62a18cec5859, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c910d27fbf3265165c7fd59609e57c82, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a258d2ecd9a494e8d56406b774da28ef, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0cf25c9917186a9e3dbd87e44f33615, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9e23742363329565223433d05d86296, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9702c569de70ef36018a26943c3226f, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebee1500f62faadfac8720740a1e527e, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62a60a0dff0c026a110e755e6c249001, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdcb3d892a0c9cf641eaedbbd9ec5937, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a1205a417c23329b514f783f4727022, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43f38f304815b66d00adc6292e217f1a, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab7cda743d5cc81945ffb7c431d6346, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0adbf1ba01f94cad79f2814d407182f, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603cf0cbb0e068a5f7727e4abab3a5d5, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba4a26e129719340e5696ac56c6b61ac, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0351c7e96a16dae18ff5a8d1664b2b1, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,362 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23280f1743db631eaa35bf3487e69147, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,362 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ceebab6d8b44ccd74ad65e871405953, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,362 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a763ac99f0e3a37f94a31b5f0479530, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,362 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eaca6c428d8bff0cd952b3c2dfe5408, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,362 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8f8bc12489e5c0370ba59c751783a23, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,362 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-09T12:16:01,362 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733746621362 2024-12-09T12:16:01,362 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733746681362 2024-12-09T12:16:01,362 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-12-09T12:16:01,363 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-09T12:16:01,363 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,363 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T12:16:01,363 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T12:16:01,363 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-09T12:16:01,363 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-09T12:16:01,363 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-09T12:16:01,363 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T12:16:01,364 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-09T12:16:01,367 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-09T12:16:01,367 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,367 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,367 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,367 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,367 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,367 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,367 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,367 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,367 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,367 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,367 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T12:16:01,368 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-09T12:16:01,368 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, REOPEN/MOVE 2024-12-09T12:16:01,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true 2024-12-09T12:16:01,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T12:16:01,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:01,370 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:01,370 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:01,370 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN 2024-12-09T12:16:01,370 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN 2024-12-09T12:16:01,370 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,370 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,370 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,370 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,370 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,370 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,370 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,370 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,370 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:01,370 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,370 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:01,370 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,370 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:01,370 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,370 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN 2024-12-09T12:16:01,370 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,370 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN 2024-12-09T12:16:01,371 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-09T12:16:01,371 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=65 2024-12-09T12:16:01,371 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f15dd41c88ad,38111,1733746548467, table=hbase:meta, region=1588230740 2024-12-09T12:16:01,372 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f15dd41c88ad,38111,1733746548467, state=OPEN 2024-12-09T12:16:01,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,374 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,374 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,374 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,374 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN to state=OFFLINE, location=null, table=test, region=62a60a0dff0c026a110e755e6c249001 to restore RIT 2024-12-09T12:16:01,375 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN to state=OFFLINE, location=null, table=test, region=ba4a26e129719340e5696ac56c6b61ac to restore RIT 2024-12-09T12:16:01,375 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN to state=OFFLINE, location=null, table=test, region=ebee1500f62faadfac8720740a1e527e to restore RIT 2024-12-09T12:16:01,375 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN to state=OFFLINE, location=null, table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8 to restore RIT 2024-12-09T12:16:01,375 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8f8bc12489e5c0370ba59c751783a23, ASSIGN to state=OFFLINE, location=null, table=test, region=c8f8bc12489e5c0370ba59c751783a23 to restore RIT 2024-12-09T12:16:01,378 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733746591378 2024-12-09T12:16:01,378 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T12:16:01,378 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,379 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-09T12:16:01,380 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-09T12:16:01,380 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN 2024-12-09T12:16:01,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,380 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc7aaad28378d11daf90725c0f9d39f5, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ac6650f2a35b2c131e93ead260da106, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8bc29ffded7f1b74a1ee2986c5e611f8, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ce28e8d0ce113d2637ab328f5ffd3be, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5251eee3971dde9f0f276cfd8632c21, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=5 2024-12-09T12:16:01,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b5306e854c5aec62eff62a18cec5859, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c910d27fbf3265165c7fd59609e57c82, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a258d2ecd9a494e8d56406b774da28ef, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0cf25c9917186a9e3dbd87e44f33615, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9e23742363329565223433d05d86296, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9702c569de70ef36018a26943c3226f, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebee1500f62faadfac8720740a1e527e, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62a60a0dff0c026a110e755e6c249001, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdcb3d892a0c9cf641eaedbbd9ec5937, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a1205a417c23329b514f783f4727022, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43f38f304815b66d00adc6292e217f1a, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab7cda743d5cc81945ffb7c431d6346, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0adbf1ba01f94cad79f2814d407182f, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603cf0cbb0e068a5f7727e4abab3a5d5, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba4a26e129719340e5696ac56c6b61ac, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0351c7e96a16dae18ff5a8d1664b2b1, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23280f1743db631eaa35bf3487e69147, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ceebab6d8b44ccd74ad65e871405953, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a763ac99f0e3a37f94a31b5f0479530, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eaca6c428d8bff0cd952b3c2dfe5408, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,391 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8f8bc12489e5c0370ba59c751783a23, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,391 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-09T12:16:01,391 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733746621391 2024-12-09T12:16:01,391 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733746681391 2024-12-09T12:16:01,391 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-12-09T12:16:01,391 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-09T12:16:01,391 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,391 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T12:16:01,391 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T12:16:01,391 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-09T12:16:01,391 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-09T12:16:01,391 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-09T12:16:01,392 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T12:16:01,393 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-09T12:16:01,395 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-09T12:16:01,395 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,395 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,395 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,395 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,395 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,395 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,395 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,395 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,395 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,395 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T12:16:01,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-09T12:16:01,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, REOPEN/MOVE 2024-12-09T12:16:01,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true 2024-12-09T12:16:01,398 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,398 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,398 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T12:16:01,398 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,398 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,398 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,398 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:01,398 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:01,398 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:01,398 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN 2024-12-09T12:16:01,398 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,398 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,398 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,398 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,398 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,398 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,398 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,398 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,399 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:01,399 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,399 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:01,399 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,399 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:01,399 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,399 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN 2024-12-09T12:16:01,399 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-09T12:16:01,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=65 2024-12-09T12:16:01,400 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f15dd41c88ad,38111,1733746548467, table=hbase:meta, region=1588230740 2024-12-09T12:16:01,400 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f15dd41c88ad,38111,1733746548467, state=OPEN 2024-12-09T12:16:01,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,404 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,405 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,407 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,407 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,409 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN to state=OFFLINE, location=null, table=test, region=62a60a0dff0c026a110e755e6c249001 to restore RIT 2024-12-09T12:16:01,410 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN to state=OFFLINE, location=null, table=test, region=ba4a26e129719340e5696ac56c6b61ac to restore RIT 2024-12-09T12:16:01,410 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN to state=OFFLINE, location=null, table=test, region=ebee1500f62faadfac8720740a1e527e to restore RIT 2024-12-09T12:16:01,410 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8, ASSIGN to state=OFFLINE, location=null, table=test, region=8bc29ffded7f1b74a1ee2986c5e611f8 to restore RIT 2024-12-09T12:16:01,416 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733746591416 2024-12-09T12:16:01,417 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T12:16:01,416 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,418 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-09T12:16:01,423 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-09T12:16:01,423 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc7aaad28378d11daf90725c0f9d39f5, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,423 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:01,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,423 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ac6650f2a35b2c131e93ead260da106, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,423 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8bc29ffded7f1b74a1ee2986c5e611f8, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,423 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,423 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ce28e8d0ce113d2637ab328f5ffd3be, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5251eee3971dde9f0f276cfd8632c21, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=5 2024-12-09T12:16:01,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b5306e854c5aec62eff62a18cec5859, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c910d27fbf3265165c7fd59609e57c82, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a258d2ecd9a494e8d56406b774da28ef, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0cf25c9917186a9e3dbd87e44f33615, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9e23742363329565223433d05d86296, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9702c569de70ef36018a26943c3226f, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebee1500f62faadfac8720740a1e527e, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62a60a0dff0c026a110e755e6c249001, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdcb3d892a0c9cf641eaedbbd9ec5937, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a1205a417c23329b514f783f4727022, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43f38f304815b66d00adc6292e217f1a, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab7cda743d5cc81945ffb7c431d6346, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0adbf1ba01f94cad79f2814d407182f, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603cf0cbb0e068a5f7727e4abab3a5d5, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba4a26e129719340e5696ac56c6b61ac, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0351c7e96a16dae18ff5a8d1664b2b1, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23280f1743db631eaa35bf3487e69147, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ceebab6d8b44ccd74ad65e871405953, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a763ac99f0e3a37f94a31b5f0479530, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,427 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eaca6c428d8bff0cd952b3c2dfe5408, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,427 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8f8bc12489e5c0370ba59c751783a23, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,427 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-09T12:16:01,427 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733746621427 2024-12-09T12:16:01,427 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733746681427 2024-12-09T12:16:01,427 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-12-09T12:16:01,427 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-09T12:16:01,427 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,427 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T12:16:01,427 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T12:16:01,428 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-09T12:16:01,428 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-09T12:16:01,428 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-09T12:16:01,428 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T12:16:01,429 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-09T12:16:01,431 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-09T12:16:01,431 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,431 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,431 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,432 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,432 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,432 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,432 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,432 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,432 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,432 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T12:16:01,433 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-09T12:16:01,433 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, REOPEN/MOVE 2024-12-09T12:16:01,434 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true 2024-12-09T12:16:01,434 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,434 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,434 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T12:16:01,434 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,434 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,434 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,434 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:01,434 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:01,435 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:01,435 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,435 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,435 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,435 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,435 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,435 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,435 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,435 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,435 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:01,435 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,435 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:01,435 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,435 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN 2024-12-09T12:16:01,435 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-09T12:16:01,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=65 2024-12-09T12:16:01,437 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f15dd41c88ad,38111,1733746548467, table=hbase:meta, region=1588230740 2024-12-09T12:16:01,437 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f15dd41c88ad,38111,1733746548467, state=OPEN 2024-12-09T12:16:01,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,440 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,440 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,441 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN to state=OFFLINE, location=null, table=test, region=62a60a0dff0c026a110e755e6c249001 to restore RIT 2024-12-09T12:16:01,441 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN to state=OFFLINE, location=null, table=test, region=ba4a26e129719340e5696ac56c6b61ac to restore RIT 2024-12-09T12:16:01,441 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ebee1500f62faadfac8720740a1e527e, ASSIGN to state=OFFLINE, location=null, table=test, region=ebee1500f62faadfac8720740a1e527e to restore RIT 2024-12-09T12:16:01,442 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,446 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733746591446 2024-12-09T12:16:01,446 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T12:16:01,446 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,448 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-09T12:16:01,454 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc7aaad28378d11daf90725c0f9d39f5, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,454 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ac6650f2a35b2c131e93ead260da106, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,454 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8bc29ffded7f1b74a1ee2986c5e611f8, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,454 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ce28e8d0ce113d2637ab328f5ffd3be, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,455 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5251eee3971dde9f0f276cfd8632c21, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=5 2024-12-09T12:16:01,455 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b5306e854c5aec62eff62a18cec5859, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,455 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c910d27fbf3265165c7fd59609e57c82, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,455 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a258d2ecd9a494e8d56406b774da28ef, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,455 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0cf25c9917186a9e3dbd87e44f33615, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,455 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9e23742363329565223433d05d86296, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9702c569de70ef36018a26943c3226f, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebee1500f62faadfac8720740a1e527e, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62a60a0dff0c026a110e755e6c249001, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdcb3d892a0c9cf641eaedbbd9ec5937, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a1205a417c23329b514f783f4727022, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43f38f304815b66d00adc6292e217f1a, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab7cda743d5cc81945ffb7c431d6346, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0adbf1ba01f94cad79f2814d407182f, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,457 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603cf0cbb0e068a5f7727e4abab3a5d5, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,457 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba4a26e129719340e5696ac56c6b61ac, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,457 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-09T12:16:01,457 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:01,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,457 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0351c7e96a16dae18ff5a8d1664b2b1, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,457 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23280f1743db631eaa35bf3487e69147, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,457 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ceebab6d8b44ccd74ad65e871405953, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,457 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,457 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a763ac99f0e3a37f94a31b5f0479530, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,457 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eaca6c428d8bff0cd952b3c2dfe5408, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,457 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8f8bc12489e5c0370ba59c751783a23, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,458 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-09T12:16:01,458 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733746621458 2024-12-09T12:16:01,458 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733746681458 2024-12-09T12:16:01,458 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-12-09T12:16:01,458 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-09T12:16:01,458 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,458 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T12:16:01,458 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T12:16:01,459 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-09T12:16:01,459 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-09T12:16:01,459 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-09T12:16:01,459 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T12:16:01,463 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 3 msec 2024-12-09T12:16:01,464 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-09T12:16:01,465 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,465 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,465 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,465 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,465 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,465 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,465 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,465 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,465 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T12:16:01,466 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-09T12:16:01,466 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, REOPEN/MOVE 2024-12-09T12:16:01,467 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true 2024-12-09T12:16:01,467 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,467 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,467 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T12:16:01,467 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,467 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,467 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,467 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:01,467 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:01,467 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,468 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,468 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,468 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,468 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,468 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,468 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,468 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,468 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:01,468 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,468 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN 2024-12-09T12:16:01,468 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-09T12:16:01,469 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=65 2024-12-09T12:16:01,469 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f15dd41c88ad,38111,1733746548467, table=hbase:meta, region=1588230740 2024-12-09T12:16:01,470 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f15dd41c88ad,38111,1733746548467, state=OPEN 2024-12-09T12:16:01,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,476 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,476 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,481 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,483 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN to state=OFFLINE, location=null, table=test, region=62a60a0dff0c026a110e755e6c249001 to restore RIT 2024-12-09T12:16:01,483 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba4a26e129719340e5696ac56c6b61ac, ASSIGN to state=OFFLINE, location=null, table=test, region=ba4a26e129719340e5696ac56c6b61ac to restore RIT 2024-12-09T12:16:01,486 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733746591486 2024-12-09T12:16:01,486 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T12:16:01,490 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,491 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-09T12:16:01,494 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc7aaad28378d11daf90725c0f9d39f5, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ac6650f2a35b2c131e93ead260da106, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8bc29ffded7f1b74a1ee2986c5e611f8, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ce28e8d0ce113d2637ab328f5ffd3be, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,497 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-09T12:16:01,497 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:01,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5251eee3971dde9f0f276cfd8632c21, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=5 2024-12-09T12:16:01,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b5306e854c5aec62eff62a18cec5859, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c910d27fbf3265165c7fd59609e57c82, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,497 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a258d2ecd9a494e8d56406b774da28ef, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0cf25c9917186a9e3dbd87e44f33615, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9e23742363329565223433d05d86296, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9702c569de70ef36018a26943c3226f, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebee1500f62faadfac8720740a1e527e, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62a60a0dff0c026a110e755e6c249001, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdcb3d892a0c9cf641eaedbbd9ec5937, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a1205a417c23329b514f783f4727022, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43f38f304815b66d00adc6292e217f1a, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab7cda743d5cc81945ffb7c431d6346, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0adbf1ba01f94cad79f2814d407182f, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603cf0cbb0e068a5f7727e4abab3a5d5, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba4a26e129719340e5696ac56c6b61ac, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0351c7e96a16dae18ff5a8d1664b2b1, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23280f1743db631eaa35bf3487e69147, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ceebab6d8b44ccd74ad65e871405953, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a763ac99f0e3a37f94a31b5f0479530, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eaca6c428d8bff0cd952b3c2dfe5408, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8f8bc12489e5c0370ba59c751783a23, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,499 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-09T12:16:01,499 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733746621499 2024-12-09T12:16:01,499 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733746681499 2024-12-09T12:16:01,499 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-12-09T12:16:01,499 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-09T12:16:01,500 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,500 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T12:16:01,500 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T12:16:01,501 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-09T12:16:01,501 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-09T12:16:01,501 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-09T12:16:01,501 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T12:16:01,504 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-12-09T12:16:01,506 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-09T12:16:01,506 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,506 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,506 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,506 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,506 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,506 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,506 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,507 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T12:16:01,508 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-09T12:16:01,508 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, REOPEN/MOVE 2024-12-09T12:16:01,509 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true 2024-12-09T12:16:01,509 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,509 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,509 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T12:16:01,509 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,509 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,509 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,509 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:01,510 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,510 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,510 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,510 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,510 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,510 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,510 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,510 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,510 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN 2024-12-09T12:16:01,510 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-09T12:16:01,515 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=65 2024-12-09T12:16:01,515 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f15dd41c88ad,38111,1733746548467, table=hbase:meta, region=1588230740 2024-12-09T12:16:01,515 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f15dd41c88ad,38111,1733746548467, state=OPEN 2024-12-09T12:16:01,515 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,518 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,518 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,518 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,519 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=62a60a0dff0c026a110e755e6c249001, ASSIGN to state=OFFLINE, location=null, table=test, region=62a60a0dff0c026a110e755e6c249001 to restore RIT 2024-12-09T12:16:01,519 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733746591519 2024-12-09T12:16:01,519 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T12:16:01,521 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,522 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-09T12:16:01,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc7aaad28378d11daf90725c0f9d39f5, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ac6650f2a35b2c131e93ead260da106, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8bc29ffded7f1b74a1ee2986c5e611f8, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ce28e8d0ce113d2637ab328f5ffd3be, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5251eee3971dde9f0f276cfd8632c21, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=5 2024-12-09T12:16:01,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b5306e854c5aec62eff62a18cec5859, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c910d27fbf3265165c7fd59609e57c82, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a258d2ecd9a494e8d56406b774da28ef, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0cf25c9917186a9e3dbd87e44f33615, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9e23742363329565223433d05d86296, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9702c569de70ef36018a26943c3226f, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebee1500f62faadfac8720740a1e527e, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62a60a0dff0c026a110e755e6c249001, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdcb3d892a0c9cf641eaedbbd9ec5937, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a1205a417c23329b514f783f4727022, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43f38f304815b66d00adc6292e217f1a, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab7cda743d5cc81945ffb7c431d6346, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0adbf1ba01f94cad79f2814d407182f, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603cf0cbb0e068a5f7727e4abab3a5d5, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba4a26e129719340e5696ac56c6b61ac, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0351c7e96a16dae18ff5a8d1664b2b1, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23280f1743db631eaa35bf3487e69147, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ceebab6d8b44ccd74ad65e871405953, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a763ac99f0e3a37f94a31b5f0479530, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eaca6c428d8bff0cd952b3c2dfe5408, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8f8bc12489e5c0370ba59c751783a23, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,530 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-09T12:16:01,530 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733746621530 2024-12-09T12:16:01,530 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733746681530 2024-12-09T12:16:01,530 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-12-09T12:16:01,530 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-09T12:16:01,531 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,531 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,531 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,780 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-09T12:16:01,781 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,781 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T12:16:01,781 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T12:16:01,781 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-09T12:16:01,781 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-09T12:16:01,782 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-09T12:16:01,782 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T12:16:01,783 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-09T12:16:01,785 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-09T12:16:01,785 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,785 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,785 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,785 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,785 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,785 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T12:16:01,786 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-09T12:16:01,786 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, REOPEN/MOVE 2024-12-09T12:16:01,787 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true 2024-12-09T12:16:01,787 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,787 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,787 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T12:16:01,787 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,787 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,787 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,787 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,787 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,787 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,787 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,787 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,787 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,787 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,787 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-09T12:16:01,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=65 2024-12-09T12:16:01,789 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f15dd41c88ad,38111,1733746548467, table=hbase:meta, region=1588230740 2024-12-09T12:16:01,789 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f15dd41c88ad,38111,1733746548467, state=OPEN 2024-12-09T12:16:01,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,794 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,795 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,795 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,798 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733746591798 2024-12-09T12:16:01,798 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T12:16:01,798 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,800 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-09T12:16:01,811 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-09T12:16:01,811 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,812 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc7aaad28378d11daf90725c0f9d39f5, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ac6650f2a35b2c131e93ead260da106, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8bc29ffded7f1b74a1ee2986c5e611f8, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ce28e8d0ce113d2637ab328f5ffd3be, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5251eee3971dde9f0f276cfd8632c21, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=5 2024-12-09T12:16:01,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b5306e854c5aec62eff62a18cec5859, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c910d27fbf3265165c7fd59609e57c82, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a258d2ecd9a494e8d56406b774da28ef, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0cf25c9917186a9e3dbd87e44f33615, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9e23742363329565223433d05d86296, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9702c569de70ef36018a26943c3226f, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebee1500f62faadfac8720740a1e527e, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62a60a0dff0c026a110e755e6c249001, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdcb3d892a0c9cf641eaedbbd9ec5937, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a1205a417c23329b514f783f4727022, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43f38f304815b66d00adc6292e217f1a, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab7cda743d5cc81945ffb7c431d6346, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0adbf1ba01f94cad79f2814d407182f, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603cf0cbb0e068a5f7727e4abab3a5d5, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba4a26e129719340e5696ac56c6b61ac, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0351c7e96a16dae18ff5a8d1664b2b1, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23280f1743db631eaa35bf3487e69147, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ceebab6d8b44ccd74ad65e871405953, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a763ac99f0e3a37f94a31b5f0479530, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eaca6c428d8bff0cd952b3c2dfe5408, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8f8bc12489e5c0370ba59c751783a23, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,817 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-09T12:16:01,817 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733746621817 2024-12-09T12:16:01,817 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733746681817 2024-12-09T12:16:01,817 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 18 msec 2024-12-09T12:16:01,817 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-09T12:16:01,817 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,817 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T12:16:01,817 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T12:16:01,817 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-09T12:16:01,817 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-09T12:16:01,817 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-09T12:16:01,817 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T12:16:01,823 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 5 msec 2024-12-09T12:16:01,825 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-09T12:16:01,826 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,826 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,826 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,826 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,826 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,826 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T12:16:01,827 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-09T12:16:01,827 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, REOPEN/MOVE 2024-12-09T12:16:01,827 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true 2024-12-09T12:16:01,828 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,828 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,828 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T12:16:01,828 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,828 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,828 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,828 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,828 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,828 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,828 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,828 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.1733746551223, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,828 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-09T12:16:01,829 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=65 2024-12-09T12:16:01,829 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f15dd41c88ad,38111,1733746548467, table=hbase:meta, region=1588230740 2024-12-09T12:16:01,829 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f15dd41c88ad,38111,1733746548467, state=OPEN 2024-12-09T12:16:01,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,839 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,839 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,840 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,846 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733746591846 2024-12-09T12:16:01,846 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T12:16:01,846 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,847 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-09T12:16:01,849 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-09T12:16:01,849 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,849 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc7aaad28378d11daf90725c0f9d39f5, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ac6650f2a35b2c131e93ead260da106, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8bc29ffded7f1b74a1ee2986c5e611f8, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ce28e8d0ce113d2637ab328f5ffd3be, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5251eee3971dde9f0f276cfd8632c21, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=5 2024-12-09T12:16:01,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b5306e854c5aec62eff62a18cec5859, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c910d27fbf3265165c7fd59609e57c82, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a258d2ecd9a494e8d56406b774da28ef, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0cf25c9917186a9e3dbd87e44f33615, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9e23742363329565223433d05d86296, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9702c569de70ef36018a26943c3226f, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebee1500f62faadfac8720740a1e527e, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62a60a0dff0c026a110e755e6c249001, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdcb3d892a0c9cf641eaedbbd9ec5937, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a1205a417c23329b514f783f4727022, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43f38f304815b66d00adc6292e217f1a, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab7cda743d5cc81945ffb7c431d6346, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0adbf1ba01f94cad79f2814d407182f, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603cf0cbb0e068a5f7727e4abab3a5d5, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba4a26e129719340e5696ac56c6b61ac, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0351c7e96a16dae18ff5a8d1664b2b1, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23280f1743db631eaa35bf3487e69147, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ceebab6d8b44ccd74ad65e871405953, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a763ac99f0e3a37f94a31b5f0479530, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eaca6c428d8bff0cd952b3c2dfe5408, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8f8bc12489e5c0370ba59c751783a23, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,853 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-09T12:16:01,853 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733746621853 2024-12-09T12:16:01,853 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733746681853 2024-12-09T12:16:01,853 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-09T12:16:01,853 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-09T12:16:01,853 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,853 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T12:16:01,853 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T12:16:01,854 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-09T12:16:01,854 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-09T12:16:01,854 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-09T12:16:01,854 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T12:16:01,855 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-09T12:16:01,856 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-09T12:16:01,856 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,856 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,856 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,856 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-09T12:16:01,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T12:16:01,857 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-09T12:16:01,857 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, REOPEN/MOVE 2024-12-09T12:16:01,857 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true 2024-12-09T12:16:01,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T12:16:01,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,858 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,858 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,858 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,858 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,858 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f15dd41c88ad,38111,1733746548467 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,858 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-09T12:16:01,859 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=65 2024-12-09T12:16:01,859 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f15dd41c88ad,38111,1733746548467, table=hbase:meta, region=1588230740 2024-12-09T12:16:01,859 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f15dd41c88ad,38111,1733746548467, state=OPEN 2024-12-09T12:16:01,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,863 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,863 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,866 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,869 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733746591869 2024-12-09T12:16:01,869 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T12:16:01,869 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,870 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-09T12:16:01,872 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-09T12:16:01,872 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T12:16:01,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,872 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc7aaad28378d11daf90725c0f9d39f5, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ac6650f2a35b2c131e93ead260da106, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8bc29ffded7f1b74a1ee2986c5e611f8, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ce28e8d0ce113d2637ab328f5ffd3be, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5251eee3971dde9f0f276cfd8632c21, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=5 2024-12-09T12:16:01,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b5306e854c5aec62eff62a18cec5859, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c910d27fbf3265165c7fd59609e57c82, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a258d2ecd9a494e8d56406b774da28ef, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0cf25c9917186a9e3dbd87e44f33615, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9e23742363329565223433d05d86296, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9702c569de70ef36018a26943c3226f, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebee1500f62faadfac8720740a1e527e, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62a60a0dff0c026a110e755e6c249001, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdcb3d892a0c9cf641eaedbbd9ec5937, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a1205a417c23329b514f783f4727022, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43f38f304815b66d00adc6292e217f1a, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab7cda743d5cc81945ffb7c431d6346, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0adbf1ba01f94cad79f2814d407182f, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603cf0cbb0e068a5f7727e4abab3a5d5, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba4a26e129719340e5696ac56c6b61ac, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0351c7e96a16dae18ff5a8d1664b2b1, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23280f1743db631eaa35bf3487e69147, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ceebab6d8b44ccd74ad65e871405953, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a763ac99f0e3a37f94a31b5f0479530, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eaca6c428d8bff0cd952b3c2dfe5408, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8f8bc12489e5c0370ba59c751783a23, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,878 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-09T12:16:01,878 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733746621878 2024-12-09T12:16:01,878 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733746681878 2024-12-09T12:16:01,878 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-09T12:16:01,879 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-09T12:16:01,879 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,879 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T12:16:01,879 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T12:16:01,879 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-09T12:16:01,879 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-09T12:16:01,879 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-09T12:16:01,879 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T12:16:01,880 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-09T12:16:01,882 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-09T12:16:01,882 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,882 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,882 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-09T12:16:01,882 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T12:16:01,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-09T12:16:01,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, REOPEN/MOVE 2024-12-09T12:16:01,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true 2024-12-09T12:16:01,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T12:16:01,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,884 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-09T12:16:01,885 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=65 2024-12-09T12:16:01,885 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f15dd41c88ad,38111,1733746548467, table=hbase:meta, region=1588230740 2024-12-09T12:16:01,885 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f15dd41c88ad,38111,1733746548467, state=OPEN 2024-12-09T12:16:01,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,887 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,887 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,887 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,888 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733746591888 2024-12-09T12:16:01,889 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T12:16:01,889 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,890 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-09T12:16:01,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc7aaad28378d11daf90725c0f9d39f5, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ac6650f2a35b2c131e93ead260da106, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,893 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-09T12:16:01,893 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8bc29ffded7f1b74a1ee2986c5e611f8, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ce28e8d0ce113d2637ab328f5ffd3be, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,893 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5251eee3971dde9f0f276cfd8632c21, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=5 2024-12-09T12:16:01,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b5306e854c5aec62eff62a18cec5859, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c910d27fbf3265165c7fd59609e57c82, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a258d2ecd9a494e8d56406b774da28ef, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0cf25c9917186a9e3dbd87e44f33615, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9e23742363329565223433d05d86296, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9702c569de70ef36018a26943c3226f, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebee1500f62faadfac8720740a1e527e, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62a60a0dff0c026a110e755e6c249001, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdcb3d892a0c9cf641eaedbbd9ec5937, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a1205a417c23329b514f783f4727022, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43f38f304815b66d00adc6292e217f1a, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab7cda743d5cc81945ffb7c431d6346, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0adbf1ba01f94cad79f2814d407182f, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603cf0cbb0e068a5f7727e4abab3a5d5, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba4a26e129719340e5696ac56c6b61ac, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0351c7e96a16dae18ff5a8d1664b2b1, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23280f1743db631eaa35bf3487e69147, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ceebab6d8b44ccd74ad65e871405953, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a763ac99f0e3a37f94a31b5f0479530, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eaca6c428d8bff0cd952b3c2dfe5408, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8f8bc12489e5c0370ba59c751783a23, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,896 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-09T12:16:01,896 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733746621896 2024-12-09T12:16:01,896 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733746681896 2024-12-09T12:16:01,896 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-09T12:16:01,896 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-09T12:16:01,896 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,896 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T12:16:01,896 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T12:16:01,896 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-09T12:16:01,896 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-09T12:16:01,896 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-09T12:16:01,896 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T12:16:01,897 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-09T12:16:01,898 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-09T12:16:01,898 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,898 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-09T12:16:01,898 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T12:16:01,899 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-09T12:16:01,899 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, REOPEN/MOVE 2024-12-09T12:16:01,900 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true 2024-12-09T12:16:01,900 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,900 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,900 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,900 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,900 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,900 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-09T12:16:01,901 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=65 2024-12-09T12:16:01,901 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f15dd41c88ad,38111,1733746548467, table=hbase:meta, region=1588230740 2024-12-09T12:16:01,901 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f15dd41c88ad,38111,1733746548467, state=OPEN 2024-12-09T12:16:01,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,904 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,904 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,904 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,910 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733746591910 2024-12-09T12:16:01,910 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T12:16:01,910 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,912 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-09T12:16:01,913 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-09T12:16:01,913 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,913 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,916 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1924) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1288) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1214) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1264) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,916 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] ipc.CallRunner(138): callId: 117 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 123 connection: 172.17.0.3:40773 deadline: 1733746571915, exception=org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false 2024-12-09T12:16:01,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc7aaad28378d11daf90725c0f9d39f5, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ac6650f2a35b2c131e93ead260da106, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8bc29ffded7f1b74a1ee2986c5e611f8, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ce28e8d0ce113d2637ab328f5ffd3be, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,917 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "f15dd41c88ad" port: 38111 start_code: 1733746548467 } transition { transition_code: OPENED region_info { region_id: 1733746552866 table_name { namespace: "default" qualifier: "test" } start_key: "zzz" end_key: "" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 82 initiating_master_active_time: 1733746548918 }; retry (#1) after 2017ms delay (Master is coming online...). org.apache.hadoop.hbase.PleaseHoldException: org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1924) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1288) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1214) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1264) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1924) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1288) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1214) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1264) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-12-09T12:16:01,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5251eee3971dde9f0f276cfd8632c21, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=5 2024-12-09T12:16:01,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b5306e854c5aec62eff62a18cec5859, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c910d27fbf3265165c7fd59609e57c82, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a258d2ecd9a494e8d56406b774da28ef, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0cf25c9917186a9e3dbd87e44f33615, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9e23742363329565223433d05d86296, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9702c569de70ef36018a26943c3226f, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebee1500f62faadfac8720740a1e527e, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62a60a0dff0c026a110e755e6c249001, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdcb3d892a0c9cf641eaedbbd9ec5937, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a1205a417c23329b514f783f4727022, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43f38f304815b66d00adc6292e217f1a, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab7cda743d5cc81945ffb7c431d6346, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0adbf1ba01f94cad79f2814d407182f, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603cf0cbb0e068a5f7727e4abab3a5d5, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba4a26e129719340e5696ac56c6b61ac, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0351c7e96a16dae18ff5a8d1664b2b1, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23280f1743db631eaa35bf3487e69147, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ceebab6d8b44ccd74ad65e871405953, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a763ac99f0e3a37f94a31b5f0479530, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eaca6c428d8bff0cd952b3c2dfe5408, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8f8bc12489e5c0370ba59c751783a23, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,919 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-09T12:16:01,919 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733746621919 2024-12-09T12:16:01,919 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733746681919 2024-12-09T12:16:01,919 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-09T12:16:01,919 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-09T12:16:01,919 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,919 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T12:16:01,919 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T12:16:01,919 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-09T12:16:01,919 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-09T12:16:01,919 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-09T12:16:01,920 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T12:16:01,921 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-09T12:16:01,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-09T12:16:01,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-09T12:16:01,922 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T12:16:01,923 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-09T12:16:01,923 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, REOPEN/MOVE 2024-12-09T12:16:01,923 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true 2024-12-09T12:16:01,924 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,924 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,924 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure f15dd41c88ad%2C33783%2C1733746548634.meta.1733746551730.meta, worker=f15dd41c88ad,36579,1733746548727 didn't hold the lock before restarting, skip acquiring lock. 2024-12-09T12:16:01,924 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-09T12:16:01,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=65 2024-12-09T12:16:01,925 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f15dd41c88ad,38111,1733746548467, table=hbase:meta, region=1588230740 2024-12-09T12:16:01,925 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f15dd41c88ad,38111,1733746548467, state=OPEN 2024-12-09T12:16:01,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,928 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,928 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,928 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,929 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733746591929 2024-12-09T12:16:01,929 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T12:16:01,929 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,931 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-09T12:16:01,932 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-09T12:16:01,932 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true 2024-12-09T12:16:01,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,932 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc7aaad28378d11daf90725c0f9d39f5, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ac6650f2a35b2c131e93ead260da106, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8bc29ffded7f1b74a1ee2986c5e611f8, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ce28e8d0ce113d2637ab328f5ffd3be, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5251eee3971dde9f0f276cfd8632c21, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=5 2024-12-09T12:16:01,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b5306e854c5aec62eff62a18cec5859, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c910d27fbf3265165c7fd59609e57c82, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a258d2ecd9a494e8d56406b774da28ef, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0cf25c9917186a9e3dbd87e44f33615, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9e23742363329565223433d05d86296, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9702c569de70ef36018a26943c3226f, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebee1500f62faadfac8720740a1e527e, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62a60a0dff0c026a110e755e6c249001, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdcb3d892a0c9cf641eaedbbd9ec5937, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a1205a417c23329b514f783f4727022, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43f38f304815b66d00adc6292e217f1a, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab7cda743d5cc81945ffb7c431d6346, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0adbf1ba01f94cad79f2814d407182f, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603cf0cbb0e068a5f7727e4abab3a5d5, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba4a26e129719340e5696ac56c6b61ac, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0351c7e96a16dae18ff5a8d1664b2b1, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23280f1743db631eaa35bf3487e69147, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ceebab6d8b44ccd74ad65e871405953, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a763ac99f0e3a37f94a31b5f0479530, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eaca6c428d8bff0cd952b3c2dfe5408, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8f8bc12489e5c0370ba59c751783a23, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,936 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-09T12:16:01,936 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733746621936 2024-12-09T12:16:01,936 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733746681936 2024-12-09T12:16:01,936 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-09T12:16:01,936 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-09T12:16:01,936 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,936 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T12:16:01,936 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T12:16:01,936 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-09T12:16:01,936 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-09T12:16:01,937 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-09T12:16:01,937 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T12:16:01,937 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-09T12:16:01,938 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42519 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,938 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "f15dd41c88ad" port: 36579 start_code: 1733746548727 } transition { transition_code: OPENED region_info { region_id: 1733746552866 table_name { namespace: "default" qualifier: "test" } start_key: "ccc" end_key: "ddd" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 76 initiating_master_active_time: 1733746548918 }; retry (#1) after 2000ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-12-09T12:16:01,939 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-09T12:16:01,939 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T12:16:01,939 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-09T12:16:01,940 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e5251eee3971dde9f0f276cfd8632c21, REOPEN/MOVE 2024-12-09T12:16:01,940 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true 2024-12-09T12:16:01,940 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-09T12:16:01,940 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-09T12:16:01,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=65 2024-12-09T12:16:01,941 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f15dd41c88ad,38111,1733746548467, table=hbase:meta, region=1588230740 2024-12-09T12:16:01,941 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f15dd41c88ad,38111,1733746548467, state=OPEN 2024-12-09T12:16:01,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T12:16:01,945 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,945 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,945 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T12:16:01,946 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733746591946 2024-12-09T12:16:01,946 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T12:16:01,946 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-09T12:16:01,947 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-09T12:16:01,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=60, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 62a60a0dff0c026a110e755e6c249001, server=f15dd41c88ad,38111,1733746548467:java.lang.RuntimeException: inject code bug; ServerCrashProcedure f15dd41c88ad,33783,1733746548634, splitWal=true, meta=true exec-time=3.1170 sec 2024-12-09T12:16:01,951 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cc7aaad28378d11daf90725c0f9d39f5, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ac6650f2a35b2c131e93ead260da106, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8bc29ffded7f1b74a1ee2986c5e611f8, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ce28e8d0ce113d2637ab328f5ffd3be, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5251eee3971dde9f0f276cfd8632c21, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=5 2024-12-09T12:16:01,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b5306e854c5aec62eff62a18cec5859, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c910d27fbf3265165c7fd59609e57c82, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a258d2ecd9a494e8d56406b774da28ef, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0cf25c9917186a9e3dbd87e44f33615, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9e23742363329565223433d05d86296, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9702c569de70ef36018a26943c3226f, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebee1500f62faadfac8720740a1e527e, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62a60a0dff0c026a110e755e6c249001, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdcb3d892a0c9cf641eaedbbd9ec5937, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a1205a417c23329b514f783f4727022, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43f38f304815b66d00adc6292e217f1a, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab7cda743d5cc81945ffb7c431d6346, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0adbf1ba01f94cad79f2814d407182f, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=603cf0cbb0e068a5f7727e4abab3a5d5, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba4a26e129719340e5696ac56c6b61ac, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b0351c7e96a16dae18ff5a8d1664b2b1, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23280f1743db631eaa35bf3487e69147, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ceebab6d8b44ccd74ad65e871405953, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a763ac99f0e3a37f94a31b5f0479530, regionState=OPEN, lastHost=f15dd41c88ad,36579,1733746548727, regionLocation=f15dd41c88ad,36579,1733746548727, openSeqNum=2 2024-12-09T12:16:01,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eaca6c428d8bff0cd952b3c2dfe5408, regionState=OPEN, lastHost=f15dd41c88ad,38111,1733746548467, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8f8bc12489e5c0370ba59c751783a23, regionState=OPENING, lastHost=f15dd41c88ad,33783,1733746548634, regionLocation=f15dd41c88ad,38111,1733746548467, openSeqNum=2 2024-12-09T12:16:01,953 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-09T12:16:01,953 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733746621953 2024-12-09T12:16:01,953 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733746681953 2024-12-09T12:16:01,953 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-09T12:16:01,976 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=346 (was 380), OpenFileDescriptor=622 (was 645), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=914 (was 898) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5010 (was 5183) 2024-12-09T12:16:01,976 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-09T12:16:01,977 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T12:16:01,977 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T12:16:01,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T12:16:01,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T12:16:01,978 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T12:16:01,978 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-09T12:16:01,978 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=164229068, stopped=false 2024-12-09T12:16:01,978 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f15dd41c88ad,42519,1733746546940 2024-12-09T12:16:01,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T12:16:01,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T12:16:01,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T12:16:01,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T12:16:01,980 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:01,981 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T12:16:01,981 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T12:16:01,981 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T12:16:01,981 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T12:16:01,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T12:16:01,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T12:16:01,982 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T12:16:01,983 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f15dd41c88ad,38111,1733746548467' ***** 2024-12-09T12:16:01,983 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T12:16:01,983 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f15dd41c88ad,36579,1733746548727' ***** 2024-12-09T12:16:01,983 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T12:16:01,983 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T12:16:01,983 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T12:16:01,983 INFO [RS:0;f15dd41c88ad:38111 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T12:16:01,983 INFO [RS:0;f15dd41c88ad:38111 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T12:16:01,983 INFO [RS:2;f15dd41c88ad:36579 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T12:16:01,983 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T12:16:01,984 INFO [RS:2;f15dd41c88ad:36579 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T12:16:01,984 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(3091): Received CLOSE for 6a763ac99f0e3a37f94a31b5f0479530 2024-12-09T12:16:01,984 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer(3091): Received CLOSE for 7eaca6c428d8bff0cd952b3c2dfe5408 2024-12-09T12:16:01,984 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T12:16:01,984 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T12:16:01,988 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(3091): Received CLOSE for c910d27fbf3265165c7fd59609e57c82 2024-12-09T12:16:01,989 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(3091): Received CLOSE for 603cf0cbb0e068a5f7727e4abab3a5d5 2024-12-09T12:16:01,989 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(3091): Received CLOSE for fab7cda743d5cc81945ffb7c431d6346 2024-12-09T12:16:01,989 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(3091): Received CLOSE for ebee1500f62faadfac8720740a1e527e 2024-12-09T12:16:01,989 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(3091): Received CLOSE for b0351c7e96a16dae18ff5a8d1664b2b1 2024-12-09T12:16:01,989 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(3091): Received CLOSE for c9702c569de70ef36018a26943c3226f 2024-12-09T12:16:01,989 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(3091): Received CLOSE for fdcb3d892a0c9cf641eaedbbd9ec5937 2024-12-09T12:16:01,989 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(3091): Received CLOSE for 43f38f304815b66d00adc6292e217f1a 2024-12-09T12:16:01,989 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(3091): Received CLOSE for 1ce28e8d0ce113d2637ab328f5ffd3be 2024-12-09T12:16:01,989 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(3091): Received CLOSE for 8ceebab6d8b44ccd74ad65e871405953 2024-12-09T12:16:01,989 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(3091): Received CLOSE for 9ac6650f2a35b2c131e93ead260da106 2024-12-09T12:16:01,989 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(959): stopping server f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:01,989 INFO [RS:2;f15dd41c88ad:36579 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T12:16:01,989 INFO [RS:2;f15dd41c88ad:36579 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;f15dd41c88ad:36579. 2024-12-09T12:16:01,989 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6a763ac99f0e3a37f94a31b5f0479530, disabling compactions & flushes 2024-12-09T12:16:01,989 DEBUG [RS:2;f15dd41c88ad:36579 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T12:16:01,989 DEBUG [RS:2;f15dd41c88ad:36579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T12:16:01,989 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1733746552866.6a763ac99f0e3a37f94a31b5f0479530. 2024-12-09T12:16:01,989 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1733746552866.6a763ac99f0e3a37f94a31b5f0479530. 2024-12-09T12:16:01,989 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1733746552866.6a763ac99f0e3a37f94a31b5f0479530. after waiting 0 ms 2024-12-09T12:16:01,990 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1733746552866.6a763ac99f0e3a37f94a31b5f0479530. 2024-12-09T12:16:01,990 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(1321): Waiting on 12 regions to close 2024-12-09T12:16:01,990 DEBUG [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(1351): Waiting on 1ce28e8d0ce113d2637ab328f5ffd3be, 43f38f304815b66d00adc6292e217f1a, 603cf0cbb0e068a5f7727e4abab3a5d5, 6a763ac99f0e3a37f94a31b5f0479530, 8bc29ffded7f1b74a1ee2986c5e611f8, 8ceebab6d8b44ccd74ad65e871405953, 9ac6650f2a35b2c131e93ead260da106, b0351c7e96a16dae18ff5a8d1664b2b1, c910d27fbf3265165c7fd59609e57c82, c9702c569de70ef36018a26943c3226f, ebee1500f62faadfac8720740a1e527e, fab7cda743d5cc81945ffb7c431d6346, fdcb3d892a0c9cf641eaedbbd9ec5937 2024-12-09T12:16:01,990 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer(3091): Received CLOSE for e5251eee3971dde9f0f276cfd8632c21 2024-12-09T12:16:01,990 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer(3091): Received CLOSE for a258d2ecd9a494e8d56406b774da28ef 2024-12-09T12:16:01,990 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer(3091): Received CLOSE for 23280f1743db631eaa35bf3487e69147 2024-12-09T12:16:01,991 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer(3091): Received CLOSE for a0cf25c9917186a9e3dbd87e44f33615 2024-12-09T12:16:01,991 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer(3091): Received CLOSE for cc7aaad28378d11daf90725c0f9d39f5 2024-12-09T12:16:01,991 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer(3091): Received CLOSE for e0adbf1ba01f94cad79f2814d407182f 2024-12-09T12:16:01,991 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer(3091): Received CLOSE for 62a60a0dff0c026a110e755e6c249001 2024-12-09T12:16:01,991 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer(3091): Received CLOSE for 3a1205a417c23329b514f783f4727022 2024-12-09T12:16:01,991 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer(3091): Received CLOSE for ba4a26e129719340e5696ac56c6b61ac 2024-12-09T12:16:01,991 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer(3091): Received CLOSE for 6b5306e854c5aec62eff62a18cec5859 2024-12-09T12:16:01,991 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer(3091): Received CLOSE for d9e23742363329565223433d05d86296 2024-12-09T12:16:01,991 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer(959): stopping server f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:01,991 INFO [RS:0;f15dd41c88ad:38111 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T12:16:01,991 INFO [RS:0;f15dd41c88ad:38111 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f15dd41c88ad:38111. 2024-12-09T12:16:01,991 DEBUG [RS:0;f15dd41c88ad:38111 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T12:16:01,991 DEBUG [RS:0;f15dd41c88ad:38111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T12:16:01,991 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T12:16:01,991 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T12:16:01,991 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T12:16:01,991 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-09T12:16:01,991 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7eaca6c428d8bff0cd952b3c2dfe5408, disabling compactions & flushes 2024-12-09T12:16:01,991 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1733746552866.7eaca6c428d8bff0cd952b3c2dfe5408. 2024-12-09T12:16:01,992 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1733746552866.7eaca6c428d8bff0cd952b3c2dfe5408. 2024-12-09T12:16:01,992 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1733746552866.7eaca6c428d8bff0cd952b3c2dfe5408. after waiting 0 ms 2024-12-09T12:16:01,992 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1733746552866.7eaca6c428d8bff0cd952b3c2dfe5408. 2024-12-09T12:16:01,992 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-12-09T12:16:01,992 DEBUG [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 23280f1743db631eaa35bf3487e69147, 3a1205a417c23329b514f783f4727022, 62a60a0dff0c026a110e755e6c249001, 6b5306e854c5aec62eff62a18cec5859, 7eaca6c428d8bff0cd952b3c2dfe5408, a0cf25c9917186a9e3dbd87e44f33615, a258d2ecd9a494e8d56406b774da28ef, ba4a26e129719340e5696ac56c6b61ac, c8f8bc12489e5c0370ba59c751783a23, cc7aaad28378d11daf90725c0f9d39f5, d9e23742363329565223433d05d86296, e0adbf1ba01f94cad79f2814d407182f, e5251eee3971dde9f0f276cfd8632c21 2024-12-09T12:16:01,992 DEBUG [RS_CLOSE_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T12:16:01,992 INFO [RS_CLOSE_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T12:16:01,992 DEBUG [RS_CLOSE_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T12:16:01,992 DEBUG [RS_CLOSE_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T12:16:01,992 DEBUG [RS_CLOSE_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T12:16:01,992 INFO [RS_CLOSE_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.65 KB heapSize=5.44 KB 2024-12-09T12:16:01,998 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/6a763ac99f0e3a37f94a31b5f0479530/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T12:16:01,998 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1733746552866.6a763ac99f0e3a37f94a31b5f0479530. 2024-12-09T12:16:01,999 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6a763ac99f0e3a37f94a31b5f0479530: Waiting for close lock at 1733746561989Running coprocessor pre-close hooks at 1733746561989Disabling compacts and flushes for region at 1733746561989Disabling writes for close at 1733746561990 (+1 ms)Writing region close event to WAL at 1733746561990Running coprocessor post-close hooks at 1733746561998 (+8 ms)Closed at 1733746561998 2024-12-09T12:16:01,999 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1733746552866.6a763ac99f0e3a37f94a31b5f0479530. 2024-12-09T12:16:01,999 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c910d27fbf3265165c7fd59609e57c82, disabling compactions & flushes 2024-12-09T12:16:01,999 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1733746552866.c910d27fbf3265165c7fd59609e57c82. 2024-12-09T12:16:01,999 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1733746552866.c910d27fbf3265165c7fd59609e57c82. 2024-12-09T12:16:01,999 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1733746552866.c910d27fbf3265165c7fd59609e57c82. after waiting 0 ms 2024-12-09T12:16:01,999 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1733746552866.c910d27fbf3265165c7fd59609e57c82. 2024-12-09T12:16:02,000 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/7eaca6c428d8bff0cd952b3c2dfe5408/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T12:16:02,001 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1733746552866.7eaca6c428d8bff0cd952b3c2dfe5408. 2024-12-09T12:16:02,001 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7eaca6c428d8bff0cd952b3c2dfe5408: Waiting for close lock at 1733746561991Running coprocessor pre-close hooks at 1733746561991Disabling compacts and flushes for region at 1733746561991Disabling writes for close at 1733746561992 (+1 ms)Writing region close event to WAL at 1733746561992Running coprocessor post-close hooks at 1733746562000 (+8 ms)Closed at 1733746562001 (+1 ms) 2024-12-09T12:16:02,001 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1733746552866.7eaca6c428d8bff0cd952b3c2dfe5408. 2024-12-09T12:16:02,001 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e5251eee3971dde9f0f276cfd8632c21, disabling compactions & flushes 2024-12-09T12:16:02,001 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21. 2024-12-09T12:16:02,001 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21. 2024-12-09T12:16:02,001 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21. after waiting 0 ms 2024-12-09T12:16:02,001 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21. 2024-12-09T12:16:02,007 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/c910d27fbf3265165c7fd59609e57c82/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T12:16:02,007 DEBUG [RS_CLOSE_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/.tmp/info/6855a3be3eb744aaaaf3b6ff960315ab is 138, key is test,ccc,1733746552866.8bc29ffded7f1b74a1ee2986c5e611f8./info:regioninfo/1733746560520/Put/seqid=0 2024-12-09T12:16:02,007 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1733746552866.c910d27fbf3265165c7fd59609e57c82. 2024-12-09T12:16:02,008 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c910d27fbf3265165c7fd59609e57c82: Waiting for close lock at 1733746561999Running coprocessor pre-close hooks at 1733746561999Disabling compacts and flushes for region at 1733746561999Disabling writes for close at 1733746561999Writing region close event to WAL at 1733746562001 (+2 ms)Running coprocessor post-close hooks at 1733746562007 (+6 ms)Closed at 1733746562007 2024-12-09T12:16:02,008 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1733746552866.c910d27fbf3265165c7fd59609e57c82. 2024-12-09T12:16:02,008 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 603cf0cbb0e068a5f7727e4abab3a5d5, disabling compactions & flushes 2024-12-09T12:16:02,008 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1733746552866.603cf0cbb0e068a5f7727e4abab3a5d5. 2024-12-09T12:16:02,008 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1733746552866.603cf0cbb0e068a5f7727e4abab3a5d5. 2024-12-09T12:16:02,008 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1733746552866.603cf0cbb0e068a5f7727e4abab3a5d5. after waiting 0 ms 2024-12-09T12:16:02,008 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1733746552866.603cf0cbb0e068a5f7727e4abab3a5d5. 2024-12-09T12:16:02,010 INFO [regionserver/f15dd41c88ad:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T12:16:02,011 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/e5251eee3971dde9f0f276cfd8632c21/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-12-09T12:16:02,012 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21. 2024-12-09T12:16:02,012 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e5251eee3971dde9f0f276cfd8632c21: Waiting for close lock at 1733746562001Running coprocessor pre-close hooks at 1733746562001Disabling compacts and flushes for region at 1733746562001Disabling writes for close at 1733746562001Writing region close event to WAL at 1733746562005 (+4 ms)Running coprocessor post-close hooks at 1733746562012 (+7 ms)Closed at 1733746562012 2024-12-09T12:16:02,012 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1733746552866.e5251eee3971dde9f0f276cfd8632c21. 2024-12-09T12:16:02,012 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a258d2ecd9a494e8d56406b774da28ef, disabling compactions & flushes 2024-12-09T12:16:02,012 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1733746552866.a258d2ecd9a494e8d56406b774da28ef. 2024-12-09T12:16:02,012 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1733746552866.a258d2ecd9a494e8d56406b774da28ef. 2024-12-09T12:16:02,012 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1733746552866.a258d2ecd9a494e8d56406b774da28ef. after waiting 0 ms 2024-12-09T12:16:02,012 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1733746552866.a258d2ecd9a494e8d56406b774da28ef. 2024-12-09T12:16:02,013 INFO [regionserver/f15dd41c88ad:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T12:16:02,020 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/603cf0cbb0e068a5f7727e4abab3a5d5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T12:16:02,021 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1733746552866.603cf0cbb0e068a5f7727e4abab3a5d5. 2024-12-09T12:16:02,021 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/a258d2ecd9a494e8d56406b774da28ef/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T12:16:02,021 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 603cf0cbb0e068a5f7727e4abab3a5d5: Waiting for close lock at 1733746562008Running coprocessor pre-close hooks at 1733746562008Disabling compacts and flushes for region at 1733746562008Disabling writes for close at 1733746562008Writing region close event to WAL at 1733746562008Running coprocessor post-close hooks at 1733746562020 (+12 ms)Closed at 1733746562021 (+1 ms) 2024-12-09T12:16:02,021 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1733746552866.603cf0cbb0e068a5f7727e4abab3a5d5. 2024-12-09T12:16:02,021 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fab7cda743d5cc81945ffb7c431d6346, disabling compactions & flushes 2024-12-09T12:16:02,021 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1733746552866.a258d2ecd9a494e8d56406b774da28ef. 2024-12-09T12:16:02,026 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1733746552866.fab7cda743d5cc81945ffb7c431d6346. 2024-12-09T12:16:02,026 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a258d2ecd9a494e8d56406b774da28ef: Waiting for close lock at 1733746562012Running coprocessor pre-close hooks at 1733746562012Disabling compacts and flushes for region at 1733746562012Disabling writes for close at 1733746562012Writing region close event to WAL at 1733746562015 (+3 ms)Running coprocessor post-close hooks at 1733746562021 (+6 ms)Closed at 1733746562021 2024-12-09T12:16:02,026 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1733746552866.fab7cda743d5cc81945ffb7c431d6346. 2024-12-09T12:16:02,027 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1733746552866.fab7cda743d5cc81945ffb7c431d6346. after waiting 0 ms 2024-12-09T12:16:02,027 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1733746552866.a258d2ecd9a494e8d56406b774da28ef. 2024-12-09T12:16:02,027 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1733746552866.fab7cda743d5cc81945ffb7c431d6346. 2024-12-09T12:16:02,027 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 23280f1743db631eaa35bf3487e69147, disabling compactions & flushes 2024-12-09T12:16:02,027 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1733746552866.23280f1743db631eaa35bf3487e69147. 2024-12-09T12:16:02,027 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1733746552866.23280f1743db631eaa35bf3487e69147. 2024-12-09T12:16:02,027 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1733746552866.23280f1743db631eaa35bf3487e69147. after waiting 0 ms 2024-12-09T12:16:02,027 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1733746552866.23280f1743db631eaa35bf3487e69147. 2024-12-09T12:16:02,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741869_1045 (size=8192) 2024-12-09T12:16:02,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741869_1045 (size=8192) 2024-12-09T12:16:02,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741869_1045 (size=8192) 2024-12-09T12:16:02,033 INFO [RS_CLOSE_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.65 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/.tmp/info/6855a3be3eb744aaaaf3b6ff960315ab 2024-12-09T12:16:02,035 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/fab7cda743d5cc81945ffb7c431d6346/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T12:16:02,035 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1733746552866.fab7cda743d5cc81945ffb7c431d6346. 2024-12-09T12:16:02,035 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fab7cda743d5cc81945ffb7c431d6346: Waiting for close lock at 1733746562021Running coprocessor pre-close hooks at 1733746562021Disabling compacts and flushes for region at 1733746562021Disabling writes for close at 1733746562027 (+6 ms)Writing region close event to WAL at 1733746562029 (+2 ms)Running coprocessor post-close hooks at 1733746562035 (+6 ms)Closed at 1733746562035 2024-12-09T12:16:02,035 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1733746552866.fab7cda743d5cc81945ffb7c431d6346. 2024-12-09T12:16:02,036 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ebee1500f62faadfac8720740a1e527e, disabling compactions & flushes 2024-12-09T12:16:02,036 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1733746552866.ebee1500f62faadfac8720740a1e527e. 2024-12-09T12:16:02,036 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1733746552866.ebee1500f62faadfac8720740a1e527e. 2024-12-09T12:16:02,036 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1733746552866.ebee1500f62faadfac8720740a1e527e. after waiting 0 ms 2024-12-09T12:16:02,036 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1733746552866.ebee1500f62faadfac8720740a1e527e. 2024-12-09T12:16:02,036 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/23280f1743db631eaa35bf3487e69147/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T12:16:02,037 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1733746552866.23280f1743db631eaa35bf3487e69147. 2024-12-09T12:16:02,037 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 23280f1743db631eaa35bf3487e69147: Waiting for close lock at 1733746562027Running coprocessor pre-close hooks at 1733746562027Disabling compacts and flushes for region at 1733746562027Disabling writes for close at 1733746562027Writing region close event to WAL at 1733746562030 (+3 ms)Running coprocessor post-close hooks at 1733746562037 (+7 ms)Closed at 1733746562037 2024-12-09T12:16:02,037 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1733746552866.23280f1743db631eaa35bf3487e69147. 2024-12-09T12:16:02,038 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a0cf25c9917186a9e3dbd87e44f33615, disabling compactions & flushes 2024-12-09T12:16:02,038 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1733746552866.a0cf25c9917186a9e3dbd87e44f33615. 2024-12-09T12:16:02,038 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1733746552866.a0cf25c9917186a9e3dbd87e44f33615. 2024-12-09T12:16:02,038 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1733746552866.a0cf25c9917186a9e3dbd87e44f33615. after waiting 0 ms 2024-12-09T12:16:02,038 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1733746552866.a0cf25c9917186a9e3dbd87e44f33615. 2024-12-09T12:16:02,042 INFO [regionserver/f15dd41c88ad:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-09T12:16:02,042 INFO [regionserver/f15dd41c88ad:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-09T12:16:02,042 DEBUG [RS_CLOSE_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/.tmp/info/6855a3be3eb744aaaaf3b6ff960315ab as hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/info/6855a3be3eb744aaaaf3b6ff960315ab 2024-12-09T12:16:02,043 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/a0cf25c9917186a9e3dbd87e44f33615/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T12:16:02,044 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/ebee1500f62faadfac8720740a1e527e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T12:16:02,044 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1733746552866.a0cf25c9917186a9e3dbd87e44f33615. 2024-12-09T12:16:02,044 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a0cf25c9917186a9e3dbd87e44f33615: Waiting for close lock at 1733746562037Running coprocessor pre-close hooks at 1733746562037Disabling compacts and flushes for region at 1733746562037Disabling writes for close at 1733746562038 (+1 ms)Writing region close event to WAL at 1733746562038Running coprocessor post-close hooks at 1733746562044 (+6 ms)Closed at 1733746562044 2024-12-09T12:16:02,044 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1733746552866.a0cf25c9917186a9e3dbd87e44f33615. 2024-12-09T12:16:02,044 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cc7aaad28378d11daf90725c0f9d39f5, disabling compactions & flushes 2024-12-09T12:16:02,045 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1733746552866.cc7aaad28378d11daf90725c0f9d39f5. 2024-12-09T12:16:02,045 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1733746552866.cc7aaad28378d11daf90725c0f9d39f5. 2024-12-09T12:16:02,045 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1733746552866.cc7aaad28378d11daf90725c0f9d39f5. after waiting 0 ms 2024-12-09T12:16:02,045 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1733746552866.cc7aaad28378d11daf90725c0f9d39f5. 2024-12-09T12:16:02,045 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1733746552866.ebee1500f62faadfac8720740a1e527e. 2024-12-09T12:16:02,045 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ebee1500f62faadfac8720740a1e527e: Waiting for close lock at 1733746562036Running coprocessor pre-close hooks at 1733746562036Disabling compacts and flushes for region at 1733746562036Disabling writes for close at 1733746562036Writing region close event to WAL at 1733746562036Running coprocessor post-close hooks at 1733746562045 (+9 ms)Closed at 1733746562045 2024-12-09T12:16:02,045 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1733746552866.ebee1500f62faadfac8720740a1e527e. 2024-12-09T12:16:02,045 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b0351c7e96a16dae18ff5a8d1664b2b1, disabling compactions & flushes 2024-12-09T12:16:02,045 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1733746552866.b0351c7e96a16dae18ff5a8d1664b2b1. 2024-12-09T12:16:02,045 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1733746552866.b0351c7e96a16dae18ff5a8d1664b2b1. 2024-12-09T12:16:02,045 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1733746552866.b0351c7e96a16dae18ff5a8d1664b2b1. after waiting 0 ms 2024-12-09T12:16:02,045 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1733746552866.b0351c7e96a16dae18ff5a8d1664b2b1. 2024-12-09T12:16:02,050 INFO [RS_CLOSE_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/info/6855a3be3eb744aaaaf3b6ff960315ab, entries=24, sequenceid=75, filesize=8 K 2024-12-09T12:16:02,051 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/cc7aaad28378d11daf90725c0f9d39f5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T12:16:02,052 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1733746552866.cc7aaad28378d11daf90725c0f9d39f5. 2024-12-09T12:16:02,052 INFO [RS_CLOSE_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.65 KB/2717, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 60ms, sequenceid=75, compaction requested=false 2024-12-09T12:16:02,052 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cc7aaad28378d11daf90725c0f9d39f5: Waiting for close lock at 1733746562044Running coprocessor pre-close hooks at 1733746562044Disabling compacts and flushes for region at 1733746562044Disabling writes for close at 1733746562045 (+1 ms)Writing region close event to WAL at 1733746562047 (+2 ms)Running coprocessor post-close hooks at 1733746562052 (+5 ms)Closed at 1733746562052 2024-12-09T12:16:02,052 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1733746552866.cc7aaad28378d11daf90725c0f9d39f5. 2024-12-09T12:16:02,053 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e0adbf1ba01f94cad79f2814d407182f, disabling compactions & flushes 2024-12-09T12:16:02,053 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1733746552866.e0adbf1ba01f94cad79f2814d407182f. 2024-12-09T12:16:02,053 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1733746552866.e0adbf1ba01f94cad79f2814d407182f. 2024-12-09T12:16:02,053 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1733746552866.e0adbf1ba01f94cad79f2814d407182f. after waiting 0 ms 2024-12-09T12:16:02,053 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1733746552866.e0adbf1ba01f94cad79f2814d407182f. 2024-12-09T12:16:02,054 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/b0351c7e96a16dae18ff5a8d1664b2b1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T12:16:02,054 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1733746552866.b0351c7e96a16dae18ff5a8d1664b2b1. 2024-12-09T12:16:02,054 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b0351c7e96a16dae18ff5a8d1664b2b1: Waiting for close lock at 1733746562045Running coprocessor pre-close hooks at 1733746562045Disabling compacts and flushes for region at 1733746562045Disabling writes for close at 1733746562045Writing region close event to WAL at 1733746562047 (+2 ms)Running coprocessor post-close hooks at 1733746562054 (+7 ms)Closed at 1733746562054 2024-12-09T12:16:02,054 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1733746552866.b0351c7e96a16dae18ff5a8d1664b2b1. 2024-12-09T12:16:02,055 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c9702c569de70ef36018a26943c3226f, disabling compactions & flushes 2024-12-09T12:16:02,055 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1733746552866.c9702c569de70ef36018a26943c3226f. 2024-12-09T12:16:02,055 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1733746552866.c9702c569de70ef36018a26943c3226f. 2024-12-09T12:16:02,055 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1733746552866.c9702c569de70ef36018a26943c3226f. after waiting 0 ms 2024-12-09T12:16:02,055 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1733746552866.c9702c569de70ef36018a26943c3226f. 2024-12-09T12:16:02,059 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/e0adbf1ba01f94cad79f2814d407182f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T12:16:02,060 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1733746552866.e0adbf1ba01f94cad79f2814d407182f. 2024-12-09T12:16:02,060 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e0adbf1ba01f94cad79f2814d407182f: Waiting for close lock at 1733746562053Running coprocessor pre-close hooks at 1733746562053Disabling compacts and flushes for region at 1733746562053Disabling writes for close at 1733746562053Writing region close event to WAL at 1733746562055 (+2 ms)Running coprocessor post-close hooks at 1733746562060 (+5 ms)Closed at 1733746562060 2024-12-09T12:16:02,060 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/c9702c569de70ef36018a26943c3226f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T12:16:02,060 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1733746552866.e0adbf1ba01f94cad79f2814d407182f. 2024-12-09T12:16:02,060 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 62a60a0dff0c026a110e755e6c249001, disabling compactions & flushes 2024-12-09T12:16:02,060 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1733746552866.62a60a0dff0c026a110e755e6c249001. 2024-12-09T12:16:02,060 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1733746552866.62a60a0dff0c026a110e755e6c249001. 2024-12-09T12:16:02,060 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1733746552866.62a60a0dff0c026a110e755e6c249001. after waiting 0 ms 2024-12-09T12:16:02,060 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1733746552866.62a60a0dff0c026a110e755e6c249001. 2024-12-09T12:16:02,061 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1733746552866.c9702c569de70ef36018a26943c3226f. 2024-12-09T12:16:02,061 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c9702c569de70ef36018a26943c3226f: Waiting for close lock at 1733746562055Running coprocessor pre-close hooks at 1733746562055Disabling compacts and flushes for region at 1733746562055Disabling writes for close at 1733746562055Writing region close event to WAL at 1733746562057 (+2 ms)Running coprocessor post-close hooks at 1733746562061 (+4 ms)Closed at 1733746562061 2024-12-09T12:16:02,061 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1733746552866.c9702c569de70ef36018a26943c3226f. 2024-12-09T12:16:02,061 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fdcb3d892a0c9cf641eaedbbd9ec5937, disabling compactions & flushes 2024-12-09T12:16:02,061 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1733746552866.fdcb3d892a0c9cf641eaedbbd9ec5937. 2024-12-09T12:16:02,061 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1733746552866.fdcb3d892a0c9cf641eaedbbd9ec5937. 2024-12-09T12:16:02,061 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1733746552866.fdcb3d892a0c9cf641eaedbbd9ec5937. after waiting 0 ms 2024-12-09T12:16:02,061 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1733746552866.fdcb3d892a0c9cf641eaedbbd9ec5937. 2024-12-09T12:16:02,070 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/62a60a0dff0c026a110e755e6c249001/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T12:16:02,070 DEBUG [RS_CLOSE_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/hbase/meta/1588230740/recovered.edits/78.seqid, newMaxSeqId=78, maxSeqId=64 2024-12-09T12:16:02,070 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/fdcb3d892a0c9cf641eaedbbd9ec5937/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T12:16:02,071 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1733746552866.62a60a0dff0c026a110e755e6c249001. 2024-12-09T12:16:02,071 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 62a60a0dff0c026a110e755e6c249001: Waiting for close lock at 1733746562060Running coprocessor pre-close hooks at 1733746562060Disabling compacts and flushes for region at 1733746562060Disabling writes for close at 1733746562060Writing region close event to WAL at 1733746562061 (+1 ms)Running coprocessor post-close hooks at 1733746562071 (+10 ms)Closed at 1733746562071 2024-12-09T12:16:02,071 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1733746552866.fdcb3d892a0c9cf641eaedbbd9ec5937. 2024-12-09T12:16:02,071 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1733746552866.62a60a0dff0c026a110e755e6c249001. 2024-12-09T12:16:02,071 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fdcb3d892a0c9cf641eaedbbd9ec5937: Waiting for close lock at 1733746562061Running coprocessor pre-close hooks at 1733746562061Disabling compacts and flushes for region at 1733746562061Disabling writes for close at 1733746562061Writing region close event to WAL at 1733746562062 (+1 ms)Running coprocessor post-close hooks at 1733746562071 (+9 ms)Closed at 1733746562071 2024-12-09T12:16:02,071 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1733746552866.fdcb3d892a0c9cf641eaedbbd9ec5937. 2024-12-09T12:16:02,071 DEBUG [RS_CLOSE_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T12:16:02,071 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3a1205a417c23329b514f783f4727022, disabling compactions & flushes 2024-12-09T12:16:02,071 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1733746552866.3a1205a417c23329b514f783f4727022. 2024-12-09T12:16:02,071 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1733746552866.3a1205a417c23329b514f783f4727022. 2024-12-09T12:16:02,071 INFO [RS_CLOSE_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T12:16:02,071 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 43f38f304815b66d00adc6292e217f1a, disabling compactions & flushes 2024-12-09T12:16:02,071 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1733746552866.3a1205a417c23329b514f783f4727022. after waiting 0 ms 2024-12-09T12:16:02,071 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1733746552866.43f38f304815b66d00adc6292e217f1a. 2024-12-09T12:16:02,071 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1733746552866.3a1205a417c23329b514f783f4727022. 2024-12-09T12:16:02,072 DEBUG [RS_CLOSE_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733746561992Running coprocessor pre-close hooks at 1733746561992Disabling compacts and flushes for region at 1733746561992Disabling writes for close at 1733746561992Obtaining lock to block concurrent updates at 1733746561992Preparing flush snapshotting stores in 1588230740 at 1733746561992Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2717, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1733746562000 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733746562001 (+1 ms)Flushing 1588230740/info: creating writer at 1733746562001Flushing 1588230740/info: appending metadata at 1733746562007 (+6 ms)Flushing 1588230740/info: closing flushed file at 1733746562007Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ff3b506: reopening flushed file at 1733746562041 (+34 ms)Finished flush of dataSize ~2.65 KB/2717, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 60ms, sequenceid=75, compaction requested=false at 1733746562052 (+11 ms)Writing region close event to WAL at 1733746562065 (+13 ms)Running coprocessor post-close hooks at 1733746562071 (+6 ms)Closed at 1733746562071 2024-12-09T12:16:02,072 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1733746552866.43f38f304815b66d00adc6292e217f1a. 2024-12-09T12:16:02,072 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1733746552866.43f38f304815b66d00adc6292e217f1a. after waiting 0 ms 2024-12-09T12:16:02,072 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1733746552866.43f38f304815b66d00adc6292e217f1a. 2024-12-09T12:16:02,072 DEBUG [RS_CLOSE_META-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-09T12:16:02,078 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/43f38f304815b66d00adc6292e217f1a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T12:16:02,079 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/3a1205a417c23329b514f783f4727022/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T12:16:02,079 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1733746552866.43f38f304815b66d00adc6292e217f1a. 2024-12-09T12:16:02,079 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 43f38f304815b66d00adc6292e217f1a: Waiting for close lock at 1733746562071Running coprocessor pre-close hooks at 1733746562071Disabling compacts and flushes for region at 1733746562071Disabling writes for close at 1733746562072 (+1 ms)Writing region close event to WAL at 1733746562074 (+2 ms)Running coprocessor post-close hooks at 1733746562079 (+5 ms)Closed at 1733746562079 2024-12-09T12:16:02,079 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1733746552866.43f38f304815b66d00adc6292e217f1a. 2024-12-09T12:16:02,079 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1ce28e8d0ce113d2637ab328f5ffd3be, disabling compactions & flushes 2024-12-09T12:16:02,079 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1733746552866.3a1205a417c23329b514f783f4727022. 2024-12-09T12:16:02,079 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1733746552866.1ce28e8d0ce113d2637ab328f5ffd3be. 2024-12-09T12:16:02,080 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1733746552866.1ce28e8d0ce113d2637ab328f5ffd3be. 2024-12-09T12:16:02,080 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3a1205a417c23329b514f783f4727022: Waiting for close lock at 1733746562071Running coprocessor pre-close hooks at 1733746562071Disabling compacts and flushes for region at 1733746562071Disabling writes for close at 1733746562071Writing region close event to WAL at 1733746562074 (+3 ms)Running coprocessor post-close hooks at 1733746562079 (+5 ms)Closed at 1733746562079 2024-12-09T12:16:02,080 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1733746552866.1ce28e8d0ce113d2637ab328f5ffd3be. after waiting 0 ms 2024-12-09T12:16:02,080 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1733746552866.1ce28e8d0ce113d2637ab328f5ffd3be. 2024-12-09T12:16:02,080 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1733746552866.3a1205a417c23329b514f783f4727022. 2024-12-09T12:16:02,080 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ba4a26e129719340e5696ac56c6b61ac, disabling compactions & flushes 2024-12-09T12:16:02,080 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1733746552866.ba4a26e129719340e5696ac56c6b61ac. 2024-12-09T12:16:02,080 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1733746552866.ba4a26e129719340e5696ac56c6b61ac. 2024-12-09T12:16:02,080 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1733746552866.ba4a26e129719340e5696ac56c6b61ac. after waiting 0 ms 2024-12-09T12:16:02,080 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1733746552866.ba4a26e129719340e5696ac56c6b61ac. 2024-12-09T12:16:02,084 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/1ce28e8d0ce113d2637ab328f5ffd3be/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T12:16:02,085 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/ba4a26e129719340e5696ac56c6b61ac/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T12:16:02,085 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1733746552866.1ce28e8d0ce113d2637ab328f5ffd3be. 2024-12-09T12:16:02,085 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1ce28e8d0ce113d2637ab328f5ffd3be: Waiting for close lock at 1733746562079Running coprocessor pre-close hooks at 1733746562079Disabling compacts and flushes for region at 1733746562079Disabling writes for close at 1733746562080 (+1 ms)Writing region close event to WAL at 1733746562080Running coprocessor post-close hooks at 1733746562085 (+5 ms)Closed at 1733746562085 2024-12-09T12:16:02,085 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1733746552866.1ce28e8d0ce113d2637ab328f5ffd3be. 2024-12-09T12:16:02,085 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8ceebab6d8b44ccd74ad65e871405953, disabling compactions & flushes 2024-12-09T12:16:02,085 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1733746552866.8ceebab6d8b44ccd74ad65e871405953. 2024-12-09T12:16:02,085 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1733746552866.ba4a26e129719340e5696ac56c6b61ac. 2024-12-09T12:16:02,085 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1733746552866.8ceebab6d8b44ccd74ad65e871405953. 2024-12-09T12:16:02,085 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ba4a26e129719340e5696ac56c6b61ac: Waiting for close lock at 1733746562080Running coprocessor pre-close hooks at 1733746562080Disabling compacts and flushes for region at 1733746562080Disabling writes for close at 1733746562080Writing region close event to WAL at 1733746562080Running coprocessor post-close hooks at 1733746562085 (+5 ms)Closed at 1733746562085 2024-12-09T12:16:02,085 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1733746552866.8ceebab6d8b44ccd74ad65e871405953. after waiting 0 ms 2024-12-09T12:16:02,085 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1733746552866.8ceebab6d8b44ccd74ad65e871405953. 2024-12-09T12:16:02,085 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1733746552866.ba4a26e129719340e5696ac56c6b61ac. 2024-12-09T12:16:02,086 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6b5306e854c5aec62eff62a18cec5859, disabling compactions & flushes 2024-12-09T12:16:02,086 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1733746552866.6b5306e854c5aec62eff62a18cec5859. 2024-12-09T12:16:02,086 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1733746552866.6b5306e854c5aec62eff62a18cec5859. 2024-12-09T12:16:02,086 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1733746552866.6b5306e854c5aec62eff62a18cec5859. after waiting 0 ms 2024-12-09T12:16:02,086 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1733746552866.6b5306e854c5aec62eff62a18cec5859. 2024-12-09T12:16:02,089 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/8ceebab6d8b44ccd74ad65e871405953/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T12:16:02,090 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/6b5306e854c5aec62eff62a18cec5859/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T12:16:02,090 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1733746552866.8ceebab6d8b44ccd74ad65e871405953. 2024-12-09T12:16:02,090 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8ceebab6d8b44ccd74ad65e871405953: Waiting for close lock at 1733746562085Running coprocessor pre-close hooks at 1733746562085Disabling compacts and flushes for region at 1733746562085Disabling writes for close at 1733746562085Writing region close event to WAL at 1733746562086 (+1 ms)Running coprocessor post-close hooks at 1733746562090 (+4 ms)Closed at 1733746562090 2024-12-09T12:16:02,090 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1733746552866.6b5306e854c5aec62eff62a18cec5859. 2024-12-09T12:16:02,090 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6b5306e854c5aec62eff62a18cec5859: Waiting for close lock at 1733746562086Running coprocessor pre-close hooks at 1733746562086Disabling compacts and flushes for region at 1733746562086Disabling writes for close at 1733746562086Writing region close event to WAL at 1733746562086Running coprocessor post-close hooks at 1733746562090 (+4 ms)Closed at 1733746562090 2024-12-09T12:16:02,090 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1733746552866.8ceebab6d8b44ccd74ad65e871405953. 2024-12-09T12:16:02,090 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1733746552866.6b5306e854c5aec62eff62a18cec5859. 2024-12-09T12:16:02,090 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d9e23742363329565223433d05d86296, disabling compactions & flushes 2024-12-09T12:16:02,090 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9ac6650f2a35b2c131e93ead260da106, disabling compactions & flushes 2024-12-09T12:16:02,090 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1733746552866.d9e23742363329565223433d05d86296. 2024-12-09T12:16:02,090 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1733746552866.9ac6650f2a35b2c131e93ead260da106. 2024-12-09T12:16:02,090 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1733746552866.d9e23742363329565223433d05d86296. 2024-12-09T12:16:02,091 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1733746552866.9ac6650f2a35b2c131e93ead260da106. 2024-12-09T12:16:02,091 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1733746552866.d9e23742363329565223433d05d86296. after waiting 0 ms 2024-12-09T12:16:02,091 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1733746552866.d9e23742363329565223433d05d86296. 2024-12-09T12:16:02,091 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1733746552866.9ac6650f2a35b2c131e93ead260da106. after waiting 0 ms 2024-12-09T12:16:02,091 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1733746552866.9ac6650f2a35b2c131e93ead260da106. 2024-12-09T12:16:02,096 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/9ac6650f2a35b2c131e93ead260da106/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T12:16:02,096 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/data/default/test/d9e23742363329565223433d05d86296/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T12:16:02,096 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1733746552866.9ac6650f2a35b2c131e93ead260da106. 2024-12-09T12:16:02,096 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9ac6650f2a35b2c131e93ead260da106: Waiting for close lock at 1733746562090Running coprocessor pre-close hooks at 1733746562090Disabling compacts and flushes for region at 1733746562090Disabling writes for close at 1733746562091 (+1 ms)Writing region close event to WAL at 1733746562091Running coprocessor post-close hooks at 1733746562096 (+5 ms)Closed at 1733746562096 2024-12-09T12:16:02,097 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1733746552866.9ac6650f2a35b2c131e93ead260da106. 2024-12-09T12:16:02,097 INFO [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1733746552866.d9e23742363329565223433d05d86296. 2024-12-09T12:16:02,097 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d9e23742363329565223433d05d86296: Waiting for close lock at 1733746562090Running coprocessor pre-close hooks at 1733746562090Disabling compacts and flushes for region at 1733746562090Disabling writes for close at 1733746562091 (+1 ms)Writing region close event to WAL at 1733746562091Running coprocessor post-close hooks at 1733746562097 (+6 ms)Closed at 1733746562097 2024-12-09T12:16:02,097 DEBUG [RS_CLOSE_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1733746552866.d9e23742363329565223433d05d86296. 2024-12-09T12:16:02,190 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(976): stopping server f15dd41c88ad,36579,1733746548727; all regions closed. 2024-12-09T12:16:02,192 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer(976): stopping server f15dd41c88ad,38111,1733746548467; all regions closed. 2024-12-09T12:16:02,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741835_1011 (size=6805) 2024-12-09T12:16:02,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741835_1011 (size=6805) 2024-12-09T12:16:02,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741835_1011 (size=6805) 2024-12-09T12:16:02,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741865_1041 (size=4201) 2024-12-09T12:16:02,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741865_1041 (size=4201) 2024-12-09T12:16:02,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741865_1041 (size=4201) 2024-12-09T12:16:02,201 DEBUG [RS:2;f15dd41c88ad:36579 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/oldWALs 2024-12-09T12:16:02,201 INFO [RS:2;f15dd41c88ad:36579 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f15dd41c88ad%2C36579%2C1733746548727:(num 1733746551262) 2024-12-09T12:16:02,201 DEBUG [RS:2;f15dd41c88ad:36579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T12:16:02,201 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T12:16:02,201 INFO [RS:2;f15dd41c88ad:36579 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T12:16:02,201 INFO [RS:2;f15dd41c88ad:36579 {}] hbase.ChoreService(370): Chore service for: regionserver/f15dd41c88ad:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-09T12:16:02,201 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T12:16:02,201 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T12:16:02,201 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T12:16:02,201 INFO [RS:2;f15dd41c88ad:36579 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T12:16:02,201 INFO [regionserver/f15dd41c88ad:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T12:16:02,202 INFO [RS:2;f15dd41c88ad:36579 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36579 2024-12-09T12:16:02,202 WARN [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2374) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T12:16:02,202 WARN [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,ccc,1733746552866.8bc29ffded7f1b74a1ee2986c5e611f8., aborting... java.io.IOException: Failed to report opened region to master: test,ccc,1733746552866.8bc29ffded7f1b74a1ee2986c5e611f8. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T12:16:02,202 ERROR [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2440): ***** ABORTING region server f15dd41c88ad,36579,1733746548727: Failed to open region test,ccc,1733746552866.8bc29ffded7f1b74a1ee2986c5e611f8. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,ccc,1733746552866.8bc29ffded7f1b74a1ee2986c5e611f8. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T12:16:02,203 ERROR [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-09T12:16:02,203 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-09T12:16:02,204 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-09T12:16:02,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f15dd41c88ad,36579,1733746548727 2024-12-09T12:16:02,204 INFO [RS:2;f15dd41c88ad:36579 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T12:16:02,205 DEBUG [RS:0;f15dd41c88ad:38111 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/oldWALs 2024-12-09T12:16:02,205 INFO [RS:0;f15dd41c88ad:38111 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f15dd41c88ad%2C38111%2C1733746548467.meta:.meta(num 1733746559468) 2024-12-09T12:16:02,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T12:16:02,206 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f15dd41c88ad,36579,1733746548727] 2024-12-09T12:16:02,207 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f15dd41c88ad,36579,1733746548727 already deleted, retry=false 2024-12-09T12:16:02,208 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f15dd41c88ad,36579,1733746548727 expired; onlineServers=1 2024-12-09T12:16:02,208 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-09T12:16:02,209 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-09T12:16:02,213 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 522047944 }, "NonHeapMemoryUsage": { "committed": 165019648, "init": 7667712, "max": -1, "used": 162364608 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "f15dd41c88ad", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 2, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 34182, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 98, "ProcessCallTime_min": 0, "ProcessCallTime_max": 35, "ProcessCallTime_mean": 5, "ProcessCallTime_25th_percentile": 8, "ProcessCallTime_median": 17, "ProcessCallTime_75th_percentile": 26, "ProcessCallTime_90th_percentile": 31, "ProcessCallTime_95th_percentile": 33, "ProcessCallTime_98th_percentile": 34, "ProcessCallTime_99th_percentile": 34, "ProcessCallTime_99.9th_percentile": 34, "ProcessCallTime_TimeRangeCount_0-1": 98, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 98, "QueueCallTime_min": 0, "QueueCallTime_max": 76, "QueueCallTime_mean": 10, "QueueCallTime_25th_percentile": 19, "QueueCallTime_median": 38, "QueueCallTime_75th_percentile": 57, "QueueCallTime_90th_percentile": 68, "QueueCallTime_95th_percentile": 72, "QueueCallTime_98th_percentile": 74, "QueueCallTime_99th_percentile": 75, "QueueCallTime_99.9th_percentile": 75, "QueueCallTime_TimeRangeCount_0-1": 98, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 98, "TotalCallTime_min": 0, "TotalCallTime_max": 80, "TotalCallTime_mean": 15, "TotalCallTime_25th_percentile": 20, "TotalCallTime_median": 40, "TotalCallTime_75th_percentile": 60, "TotalCallTime_90th_percentile": 72, "TotalCallTime_95th_percentile": 76, "TotalCallTime_98th_percentile": 78, "TotalCallTime_99th_percentile": 79, "TotalCallTime_99.9th_percentile": 79, "TotalCallTime_TimeRangeCount_0-1": 98, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 98, "ResponseSize_min": 0, "ResponseSize_max": 22358, "ResponseSize_mean": 697, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16768, "ResponseSize_90th_percentile": 20122, "ResponseSize_95th_percentile": 21240, "ResponseSize_98th_percentile": 21910, "ResponseSize_99th_percentile": 22134, "ResponseSize_99.9th_percentile": 22335, "ResponseSize_SizeRangeCount_0-10": 98, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 98, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 316, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 98, "sentBytes": 54700 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "f15dd41c88ad", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:60469", "tag.serverName": "f15dd41c88ad,38111,1733746548467", "tag.clusterId": "bd14337c-7e24-4686-a8c1-6c16b6d09ea0", "tag.Context": "regionserver", "tag.Hostname": "f15dd41c88ad", "regionCount": 8, "storeCount": 8, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2048, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733746548467, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 0, "localBytesRead": 0, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 0, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 8, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 8, "rpcScanRequestCount": 8, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 8, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 8, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 8, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 8, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-09T12:16:02,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741833_1009 (size=6795) 2024-12-09T12:16:02,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741833_1009 (size=6795) 2024-12-09T12:16:02,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741833_1009 (size=6795) 2024-12-09T12:16:02,233 DEBUG [RS:0;f15dd41c88ad:38111 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/oldWALs 2024-12-09T12:16:02,233 INFO [RS:0;f15dd41c88ad:38111 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f15dd41c88ad%2C38111%2C1733746548467:(num 1733746551067) 2024-12-09T12:16:02,233 DEBUG [RS:0;f15dd41c88ad:38111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T12:16:02,233 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T12:16:02,233 INFO [RS:0;f15dd41c88ad:38111 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T12:16:02,233 INFO [RS:0;f15dd41c88ad:38111 {}] hbase.ChoreService(370): Chore service for: regionserver/f15dd41c88ad:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-09T12:16:02,233 INFO [RS:0;f15dd41c88ad:38111 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T12:16:02,233 INFO [regionserver/f15dd41c88ad:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T12:16:02,233 WARN [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2374) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T12:16:02,234 INFO [RS:0;f15dd41c88ad:38111 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38111 2024-12-09T12:16:02,234 WARN [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,zzz,1733746552866.c8f8bc12489e5c0370ba59c751783a23., aborting... java.io.IOException: Failed to report opened region to master: test,zzz,1733746552866.c8f8bc12489e5c0370ba59c751783a23. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T12:16:02,234 ERROR [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2440): ***** ABORTING region server f15dd41c88ad,38111,1733746548467: Failed to open region test,zzz,1733746552866.c8f8bc12489e5c0370ba59c751783a23. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,zzz,1733746552866.c8f8bc12489e5c0370ba59c751783a23. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T12:16:02,234 ERROR [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-09T12:16:02,236 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-09T12:16:02,236 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-09T12:16:02,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f15dd41c88ad,38111,1733746548467 2024-12-09T12:16:02,237 INFO [RS:0;f15dd41c88ad:38111 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T12:16:02,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T12:16:02,238 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-09T12:16:02,239 DEBUG [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-09T12:16:02,239 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f15dd41c88ad,38111,1733746548467] 2024-12-09T12:16:02,241 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f15dd41c88ad,38111,1733746548467 already deleted, retry=false 2024-12-09T12:16:02,241 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f15dd41c88ad,38111,1733746548467 expired; onlineServers=0 2024-12-09T12:16:02,241 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f15dd41c88ad,42519,1733746546940' ***** 2024-12-09T12:16:02,241 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-09T12:16:02,241 INFO [M:0;f15dd41c88ad:42519 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T12:16:02,241 INFO [M:0;f15dd41c88ad:42519 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T12:16:02,241 DEBUG [M:0;f15dd41c88ad:42519 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-09T12:16:02,242 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-09T12:16:02,242 DEBUG [M:0;f15dd41c88ad:42519 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-09T12:16:02,242 INFO [M:0;f15dd41c88ad:42519 {}] hbase.ChoreService(370): Chore service for: master/f15dd41c88ad:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-09T12:16:02,242 INFO [RS_OPEN_REGION-regionserver/f15dd41c88ad:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 524145096 }, "NonHeapMemoryUsage": { "committed": 165216256, "init": 7667712, "max": -1, "used": 162540992 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "f15dd41c88ad", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 2, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 34182, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 98, "ProcessCallTime_min": 0, "ProcessCallTime_max": 35, "ProcessCallTime_mean": 5, "ProcessCallTime_25th_percentile": 8, "ProcessCallTime_median": 17, "ProcessCallTime_75th_percentile": 26, "ProcessCallTime_90th_percentile": 31, "ProcessCallTime_95th_percentile": 33, "ProcessCallTime_98th_percentile": 34, "ProcessCallTime_99th_percentile": 34, "ProcessCallTime_99.9th_percentile": 34, "ProcessCallTime_TimeRangeCount_0-1": 98, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 98, "QueueCallTime_min": 0, "QueueCallTime_max": 76, "QueueCallTime_mean": 10, "QueueCallTime_25th_percentile": 19, "QueueCallTime_median": 38, "QueueCallTime_75th_percentile": 57, "QueueCallTime_90th_percentile": 68, "QueueCallTime_95th_percentile": 72, "QueueCallTime_98th_percentile": 74, "QueueCallTime_99th_percentile": 75, "QueueCallTime_99.9th_percentile": 75, "QueueCallTime_TimeRangeCount_0-1": 98, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 98, "TotalCallTime_min": 0, "TotalCallTime_max": 80, "TotalCallTime_mean": 15, "TotalCallTime_25th_percentile": 20, "TotalCallTime_median": 40, "TotalCallTime_75th_percentile": 60, "TotalCallTime_90th_percentile": 72, "TotalCallTime_95th_percentile": 76, "TotalCallTime_98th_percentile": 78, "TotalCallTime_99th_percentile": 79, "TotalCallTime_99.9th_percentile": 79, "TotalCallTime_TimeRangeCount_0-1": 98, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 98, "ResponseSize_min": 0, "ResponseSize_max": 22358, "ResponseSize_mean": 697, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16768, "ResponseSize_90th_percentile": 20122, "ResponseSize_95th_percentile": 21240, "ResponseSize_98th_percentile": 21910, "ResponseSize_99th_percentile": 22134, "ResponseSize_99.9th_percentile": 22335, "ResponseSize_SizeRangeCount_0-10": 98, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 98, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 316, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 98, "sentBytes": 54700 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "f15dd41c88ad", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:60469", "tag.serverName": "f15dd41c88ad,38111,1733746548467", "tag.clusterId": "bd14337c-7e24-4686-a8c1-6c16b6d09ea0", "tag.Context": "regionserver", "tag.Hostname": "f15dd41c88ad", "regionCount": 8, "storeCount": 8, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2048, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733746548467, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 0, "localBytesRead": 0, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 0, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 8, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 8, "rpcScanRequestCount": 8, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 8, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 8, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 8, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 8, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-09T12:16:02,242 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster-HFileCleaner.large.0-1733746550660 {}] cleaner.HFileCleaner(306): Exit Thread[master/f15dd41c88ad:0:becomeActiveMaster-HFileCleaner.large.0-1733746550660,5,FailOnTimeoutGroup] 2024-12-09T12:16:02,242 DEBUG [master/f15dd41c88ad:0:becomeActiveMaster-HFileCleaner.small.0-1733746550670 {}] cleaner.HFileCleaner(306): Exit Thread[master/f15dd41c88ad:0:becomeActiveMaster-HFileCleaner.small.0-1733746550670,5,FailOnTimeoutGroup] 2024-12-09T12:16:02,242 INFO [M:0;f15dd41c88ad:42519 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T12:16:02,243 DEBUG [M:0;f15dd41c88ad:42519 {}] master.HMaster(1795): Stopping service threads 2024-12-09T12:16:02,243 INFO [M:0;f15dd41c88ad:42519 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-09T12:16:02,243 INFO [M:0;f15dd41c88ad:42519 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T12:16:02,243 INFO [M:0;f15dd41c88ad:42519 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T12:16:02,244 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-09T12:16:02,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-09T12:16:02,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T12:16:02,251 DEBUG [M:0;f15dd41c88ad:42519 {}] zookeeper.ZKUtil(347): master:42519-0x1012b1c64ac0000, quorum=127.0.0.1:60469, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-09T12:16:02,251 WARN [M:0;f15dd41c88ad:42519 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-09T12:16:02,255 INFO [M:0;f15dd41c88ad:42519 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/.lastflushedseqids 2024-12-09T12:16:02,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741870_1046 (size=880) 2024-12-09T12:16:02,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741870_1046 (size=880) 2024-12-09T12:16:02,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741870_1046 (size=880) 2024-12-09T12:16:02,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T12:16:02,315 INFO [RS:2;f15dd41c88ad:36579 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T12:16:02,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36579-0x1012b1c64ac0003, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T12:16:02,315 INFO [RS:2;f15dd41c88ad:36579 {}] regionserver.HRegionServer(1031): Exiting; stopping=f15dd41c88ad,36579,1733746548727; zookeeper connection closed. 2024-12-09T12:16:02,315 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5ad6efd1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5ad6efd1 2024-12-09T12:16:02,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T12:16:02,340 INFO [RS:0;f15dd41c88ad:38111 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T12:16:02,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38111-0x1012b1c64ac0001, quorum=127.0.0.1:60469, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T12:16:02,340 INFO [RS:0;f15dd41c88ad:38111 {}] regionserver.HRegionServer(1031): Exiting; stopping=f15dd41c88ad,38111,1733746548467; zookeeper connection closed. 2024-12-09T12:16:02,340 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@22956944 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@22956944 2024-12-09T12:16:02,341 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-09T12:16:02,691 INFO [M:0;f15dd41c88ad:42519 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T12:16:02,692 INFO [M:0;f15dd41c88ad:42519 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-09T12:16:02,692 DEBUG [M:0;f15dd41c88ad:42519 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T12:16:02,692 INFO [M:0;f15dd41c88ad:42519 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T12:16:02,692 DEBUG [M:0;f15dd41c88ad:42519 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T12:16:02,692 DEBUG [M:0;f15dd41c88ad:42519 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T12:16:02,692 DEBUG [M:0;f15dd41c88ad:42519 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T12:16:02,693 INFO [M:0;f15dd41c88ad:42519 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=259.50 KB heapSize=315.41 KB 2024-12-09T12:16:02,716 DEBUG [M:0;f15dd41c88ad:42519 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9dc64d8fb86144d39fec2fc9cd794706 is 82, key is hbase:meta,,1/info:regioninfo/1733746559937/Put/seqid=0 2024-12-09T12:16:02,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33961 is added to blk_1073741871_1047 (size=6071) 2024-12-09T12:16:02,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741871_1047 (size=6071) 2024-12-09T12:16:02,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39373 is added to blk_1073741871_1047 (size=6071) 2024-12-09T12:16:02,739 INFO [M:0;f15dd41c88ad:42519 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9dc64d8fb86144d39fec2fc9cd794706 2024-12-09T12:16:02,768 DEBUG [M:0;f15dd41c88ad:42519 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36283/user/jenkins/test-data/e050ade5-f530-f927-5b51-3263bde0d5ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b3c15b30c7bc4726ba8dad0a3ea5ac06 is 2987, key is \x00\x00\x00\x00\x00\x00\x00