2024-11-21 08:13:23,746 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-21 08:13:23,764 main DEBUG Took 0.015569 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-21 08:13:23,765 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-21 08:13:23,766 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-21 08:13:23,767 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-21 08:13:23,769 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 08:13:23,779 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-21 08:13:23,795 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 08:13:23,797 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 08:13:23,797 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 08:13:23,798 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 08:13:23,799 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 08:13:23,799 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 08:13:23,800 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 08:13:23,801 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 08:13:23,802 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 08:13:23,802 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 08:13:23,803 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 08:13:23,803 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 08:13:23,804 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 08:13:23,804 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 08:13:23,805 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 08:13:23,806 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 08:13:23,806 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 08:13:23,807 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 08:13:23,807 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 08:13:23,808 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 08:13:23,808 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 08:13:23,808 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 08:13:23,809 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 08:13:23,809 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 08:13:23,810 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 08:13:23,810 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-21 08:13:23,811 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 08:13:23,813 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-21 08:13:23,815 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-21 08:13:23,816 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-21 08:13:23,818 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-21 08:13:23,819 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-21 08:13:23,831 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-21 08:13:23,836 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-21 08:13:23,838 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-21 08:13:23,839 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-21 08:13:23,840 main DEBUG createAppenders(={Console}) 2024-11-21 08:13:23,841 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-21 08:13:23,842 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-21 08:13:23,842 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-21 08:13:23,843 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-21 08:13:23,843 main DEBUG OutputStream closed 2024-11-21 08:13:23,844 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-21 08:13:23,844 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-21 08:13:23,845 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-21 08:13:23,951 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-21 08:13:23,954 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-21 08:13:23,956 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-21 08:13:23,957 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-21 08:13:23,958 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-21 08:13:23,958 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-21 08:13:23,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-21 08:13:23,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-21 08:13:23,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-21 08:13:23,960 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-21 08:13:23,960 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-21 08:13:23,961 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-21 08:13:23,962 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-21 08:13:23,962 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-21 08:13:23,963 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-21 08:13:23,963 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-21 08:13:23,964 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-21 08:13:23,965 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-21 08:13:23,969 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21 08:13:23,970 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-21 08:13:23,970 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-21 08:13:23,971 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-21T08:13:23,992 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-21 08:13:23,996 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-21 08:13:23,997 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21T08:13:24,296 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7 2024-11-21T08:13:24,322 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-21T08:13:24,338 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/cluster_32f8faac-9813-3576-7230-c250ea97bb11, deleteOnExit=true 2024-11-21T08:13:24,339 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-21T08:13:24,340 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/test.cache.data in system properties and HBase conf 2024-11-21T08:13:24,341 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/hadoop.tmp.dir in system properties and HBase conf 2024-11-21T08:13:24,341 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/hadoop.log.dir in system properties and HBase conf 2024-11-21T08:13:24,342 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-21T08:13:24,343 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-21T08:13:24,343 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-21T08:13:24,459 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-21T08:13:24,583 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T08:13:24,590 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-21T08:13:24,591 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-21T08:13:24,592 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-21T08:13:24,593 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T08:13:24,594 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-21T08:13:24,595 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-21T08:13:24,595 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T08:13:24,596 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T08:13:24,597 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-21T08:13:24,598 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/nfs.dump.dir in system properties and HBase conf 2024-11-21T08:13:24,598 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/java.io.tmpdir in system properties and HBase conf 2024-11-21T08:13:24,599 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T08:13:24,599 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-21T08:13:24,600 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-21T08:13:26,410 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-21T08:13:26,539 INFO [Time-limited test {}] log.Log(170): Logging initialized @3613ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-21T08:13:26,645 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T08:13:26,765 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T08:13:26,839 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T08:13:26,840 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T08:13:26,843 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-21T08:13:26,870 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T08:13:26,875 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/hadoop.log.dir/,AVAILABLE} 2024-11-21T08:13:26,876 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T08:13:27,144 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64418798{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/java.io.tmpdir/jetty-localhost-36973-hadoop-hdfs-3_4_1-tests_jar-_-any-6472612375159328190/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-21T08:13:27,169 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:36973} 2024-11-21T08:13:27,169 INFO [Time-limited test {}] server.Server(415): Started @4244ms 2024-11-21T08:13:28,426 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T08:13:28,438 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T08:13:28,447 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T08:13:28,448 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T08:13:28,449 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-21T08:13:28,450 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/hadoop.log.dir/,AVAILABLE} 2024-11-21T08:13:28,451 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T08:13:28,602 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@79b30fb0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/java.io.tmpdir/jetty-localhost-42733-hadoop-hdfs-3_4_1-tests_jar-_-any-6498548448535558162/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T08:13:28,603 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:42733} 2024-11-21T08:13:28,604 INFO [Time-limited test {}] server.Server(415): Started @5678ms 2024-11-21T08:13:28,684 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-21T08:13:28,826 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T08:13:28,833 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T08:13:28,840 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T08:13:28,841 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T08:13:28,841 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-21T08:13:28,849 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/hadoop.log.dir/,AVAILABLE} 2024-11-21T08:13:28,850 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T08:13:28,980 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@34fcb585{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/java.io.tmpdir/jetty-localhost-37519-hadoop-hdfs-3_4_1-tests_jar-_-any-6993163793545428892/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T08:13:28,981 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:37519} 2024-11-21T08:13:28,981 INFO [Time-limited test {}] server.Server(415): Started @6056ms 2024-11-21T08:13:28,984 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-21T08:13:29,067 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T08:13:29,076 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T08:13:29,078 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T08:13:29,078 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T08:13:29,079 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-21T08:13:29,085 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/hadoop.log.dir/,AVAILABLE} 2024-11-21T08:13:29,086 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T08:13:29,212 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3c1f701{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/java.io.tmpdir/jetty-localhost-32809-hadoop-hdfs-3_4_1-tests_jar-_-any-9061365163869951933/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T08:13:29,213 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:32809} 2024-11-21T08:13:29,214 INFO [Time-limited test {}] server.Server(415): Started @6288ms 2024-11-21T08:13:29,217 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-21T08:13:30,571 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/cluster_32f8faac-9813-3576-7230-c250ea97bb11/data/data2/current/BP-1596423341-172.17.0.3-1732176805302/current, will proceed with Du for space computation calculation, 2024-11-21T08:13:30,571 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/cluster_32f8faac-9813-3576-7230-c250ea97bb11/data/data3/current/BP-1596423341-172.17.0.3-1732176805302/current, will proceed with Du for space computation calculation, 2024-11-21T08:13:30,571 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/cluster_32f8faac-9813-3576-7230-c250ea97bb11/data/data1/current/BP-1596423341-172.17.0.3-1732176805302/current, will proceed with Du for space computation calculation, 2024-11-21T08:13:30,571 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/cluster_32f8faac-9813-3576-7230-c250ea97bb11/data/data4/current/BP-1596423341-172.17.0.3-1732176805302/current, will proceed with Du for space computation calculation, 2024-11-21T08:13:30,607 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-21T08:13:30,607 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-21T08:13:30,652 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfd91d7ba09a5107f with lease ID 0x38b13a94e15208dd: Processing first storage report for DS-7db7df2d-be4b-495c-9b4b-5548750637bb from datanode DatanodeRegistration(127.0.0.1:33295, datanodeUuid=5eb3583b-71b6-4d81-aaa0-0211a5080bbf, infoPort=46533, infoSecurePort=0, ipcPort=42789, storageInfo=lv=-57;cid=testClusterID;nsid=589776541;c=1732176805302) 2024-11-21T08:13:30,653 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfd91d7ba09a5107f with lease ID 0x38b13a94e15208dd: from storage DS-7db7df2d-be4b-495c-9b4b-5548750637bb node DatanodeRegistration(127.0.0.1:33295, datanodeUuid=5eb3583b-71b6-4d81-aaa0-0211a5080bbf, infoPort=46533, infoSecurePort=0, ipcPort=42789, storageInfo=lv=-57;cid=testClusterID;nsid=589776541;c=1732176805302), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-21T08:13:30,653 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe33b1182e1e37d3d with lease ID 0x38b13a94e15208dc: Processing first storage report for DS-f098b1c9-3781-437c-aa42-0db9fe2ba16f from datanode DatanodeRegistration(127.0.0.1:34217, datanodeUuid=6b7537cb-8085-4505-a175-e79f5c1a7652, infoPort=37401, infoSecurePort=0, ipcPort=38251, storageInfo=lv=-57;cid=testClusterID;nsid=589776541;c=1732176805302) 2024-11-21T08:13:30,653 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe33b1182e1e37d3d with lease ID 0x38b13a94e15208dc: from storage DS-f098b1c9-3781-437c-aa42-0db9fe2ba16f node DatanodeRegistration(127.0.0.1:34217, datanodeUuid=6b7537cb-8085-4505-a175-e79f5c1a7652, infoPort=37401, infoSecurePort=0, ipcPort=38251, storageInfo=lv=-57;cid=testClusterID;nsid=589776541;c=1732176805302), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-21T08:13:30,654 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfd91d7ba09a5107f with lease ID 0x38b13a94e15208dd: Processing first storage report for DS-60247027-c321-40bd-849a-3d3ed9d75afe from datanode DatanodeRegistration(127.0.0.1:33295, datanodeUuid=5eb3583b-71b6-4d81-aaa0-0211a5080bbf, infoPort=46533, infoSecurePort=0, ipcPort=42789, storageInfo=lv=-57;cid=testClusterID;nsid=589776541;c=1732176805302) 2024-11-21T08:13:30,654 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfd91d7ba09a5107f with lease ID 0x38b13a94e15208dd: from storage DS-60247027-c321-40bd-849a-3d3ed9d75afe node DatanodeRegistration(127.0.0.1:33295, datanodeUuid=5eb3583b-71b6-4d81-aaa0-0211a5080bbf, infoPort=46533, infoSecurePort=0, ipcPort=42789, storageInfo=lv=-57;cid=testClusterID;nsid=589776541;c=1732176805302), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-21T08:13:30,654 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe33b1182e1e37d3d with lease ID 0x38b13a94e15208dc: Processing first storage report for DS-a9ab0c58-ca75-4e11-89b0-12338c5e577f from datanode DatanodeRegistration(127.0.0.1:34217, datanodeUuid=6b7537cb-8085-4505-a175-e79f5c1a7652, infoPort=37401, infoSecurePort=0, ipcPort=38251, storageInfo=lv=-57;cid=testClusterID;nsid=589776541;c=1732176805302) 2024-11-21T08:13:30,654 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe33b1182e1e37d3d with lease ID 0x38b13a94e15208dc: from storage DS-a9ab0c58-ca75-4e11-89b0-12338c5e577f node DatanodeRegistration(127.0.0.1:34217, datanodeUuid=6b7537cb-8085-4505-a175-e79f5c1a7652, infoPort=37401, infoSecurePort=0, ipcPort=38251, storageInfo=lv=-57;cid=testClusterID;nsid=589776541;c=1732176805302), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-21T08:13:30,676 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/cluster_32f8faac-9813-3576-7230-c250ea97bb11/data/data5/current/BP-1596423341-172.17.0.3-1732176805302/current, will proceed with Du for space computation calculation, 2024-11-21T08:13:30,676 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/cluster_32f8faac-9813-3576-7230-c250ea97bb11/data/data6/current/BP-1596423341-172.17.0.3-1732176805302/current, will proceed with Du for space computation calculation, 2024-11-21T08:13:30,697 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-21T08:13:30,702 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd3efe873392f4cd1 with lease ID 0x38b13a94e15208de: Processing first storage report for DS-9db7f2dc-3c49-4f47-8420-8e992a2830c6 from datanode DatanodeRegistration(127.0.0.1:43379, datanodeUuid=4f3f464b-3066-494b-bd48-e5b09098f07f, infoPort=45803, infoSecurePort=0, ipcPort=45259, storageInfo=lv=-57;cid=testClusterID;nsid=589776541;c=1732176805302) 2024-11-21T08:13:30,702 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd3efe873392f4cd1 with lease ID 0x38b13a94e15208de: from storage DS-9db7f2dc-3c49-4f47-8420-8e992a2830c6 node DatanodeRegistration(127.0.0.1:43379, datanodeUuid=4f3f464b-3066-494b-bd48-e5b09098f07f, infoPort=45803, infoSecurePort=0, ipcPort=45259, storageInfo=lv=-57;cid=testClusterID;nsid=589776541;c=1732176805302), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-21T08:13:30,703 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd3efe873392f4cd1 with lease ID 0x38b13a94e15208de: Processing first storage report for DS-6d9e26df-098b-4138-bd17-63d501448ca7 from datanode DatanodeRegistration(127.0.0.1:43379, datanodeUuid=4f3f464b-3066-494b-bd48-e5b09098f07f, infoPort=45803, infoSecurePort=0, ipcPort=45259, storageInfo=lv=-57;cid=testClusterID;nsid=589776541;c=1732176805302) 2024-11-21T08:13:30,703 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd3efe873392f4cd1 with lease ID 0x38b13a94e15208de: from storage DS-6d9e26df-098b-4138-bd17-63d501448ca7 node DatanodeRegistration(127.0.0.1:43379, datanodeUuid=4f3f464b-3066-494b-bd48-e5b09098f07f, infoPort=45803, infoSecurePort=0, ipcPort=45259, storageInfo=lv=-57;cid=testClusterID;nsid=589776541;c=1732176805302), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-21T08:13:30,712 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7 2024-11-21T08:13:30,782 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/cluster_32f8faac-9813-3576-7230-c250ea97bb11/zookeeper_0, clientPort=52614, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/cluster_32f8faac-9813-3576-7230-c250ea97bb11/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f28ab09e-aa90-f5dc-70a9-5bde8996b3d7/cluster_32f8faac-9813-3576-7230-c250ea97bb11/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-21T08:13:30,795 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52614 2024-11-21T08:13:30,809 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T08:13:30,813 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T08:13:31,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741825_1001 (size=7) 2024-11-21T08:13:31,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741825_1001 (size=7) 2024-11-21T08:13:31,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741825_1001 (size=7) 2024-11-21T08:13:31,447 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db with version=8 2024-11-21T08:13:31,447 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/hbase-staging 2024-11-21T08:13:31,525 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-21T08:13:31,771 INFO [Time-limited test {}] client.ConnectionUtils(128): master/6ed56fb6cbf7:0 server-side Connection retries=45 2024-11-21T08:13:31,784 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T08:13:31,784 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T08:13:31,791 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T08:13:31,792 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T08:13:31,792 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T08:13:31,999 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-21T08:13:32,106 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-21T08:13:32,118 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-21T08:13:32,123 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T08:13:32,160 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1595 (auto-detected) 2024-11-21T08:13:32,162 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-21T08:13:32,195 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:32901 2024-11-21T08:13:32,227 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:32901 connecting to ZooKeeper ensemble=127.0.0.1:52614 2024-11-21T08:13:32,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:329010x0, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T08:13:32,334 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:32901-0x1015c748eb50000 connected 2024-11-21T08:13:32,471 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T08:13:32,482 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T08:13:32,504 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T08:13:32,509 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db, hbase.cluster.distributed=false 2024-11-21T08:13:32,555 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T08:13:32,564 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32901 2024-11-21T08:13:32,567 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32901 2024-11-21T08:13:32,571 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32901 2024-11-21T08:13:32,572 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32901 2024-11-21T08:13:32,576 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32901 2024-11-21T08:13:32,742 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6ed56fb6cbf7:0 server-side Connection retries=45 2024-11-21T08:13:32,745 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T08:13:32,746 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T08:13:32,746 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T08:13:32,746 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T08:13:32,747 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T08:13:32,751 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T08:13:32,755 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T08:13:32,757 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35295 2024-11-21T08:13:32,759 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35295 connecting to ZooKeeper ensemble=127.0.0.1:52614 2024-11-21T08:13:32,761 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T08:13:32,766 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T08:13:32,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:352950x0, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T08:13:32,795 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:352950x0, quorum=127.0.0.1:52614, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T08:13:32,801 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T08:13:32,804 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35295-0x1015c748eb50001 connected 2024-11-21T08:13:32,817 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T08:13:32,823 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T08:13:32,831 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T08:13:32,836 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35295 2024-11-21T08:13:32,837 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35295 2024-11-21T08:13:32,840 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35295 2024-11-21T08:13:32,843 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35295 2024-11-21T08:13:32,844 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35295 2024-11-21T08:13:32,866 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6ed56fb6cbf7:0 server-side Connection retries=45 2024-11-21T08:13:32,867 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T08:13:32,867 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T08:13:32,868 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T08:13:32,868 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T08:13:32,868 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T08:13:32,868 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T08:13:32,868 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T08:13:32,869 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38845 2024-11-21T08:13:32,872 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38845 connecting to ZooKeeper ensemble=127.0.0.1:52614 2024-11-21T08:13:32,874 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T08:13:32,882 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T08:13:32,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:388450x0, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T08:13:32,902 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:388450x0, quorum=127.0.0.1:52614, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T08:13:32,908 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T08:13:32,908 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38845-0x1015c748eb50002 connected 2024-11-21T08:13:32,921 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T08:13:32,924 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T08:13:32,927 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T08:13:32,932 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38845 2024-11-21T08:13:32,938 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38845 2024-11-21T08:13:32,939 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38845 2024-11-21T08:13:32,940 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38845 2024-11-21T08:13:32,943 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38845 2024-11-21T08:13:32,969 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6ed56fb6cbf7:0 server-side Connection retries=45 2024-11-21T08:13:32,970 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T08:13:32,970 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T08:13:32,971 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T08:13:32,971 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T08:13:32,971 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T08:13:32,972 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T08:13:32,972 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T08:13:32,973 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:33091 2024-11-21T08:13:32,976 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33091 connecting to ZooKeeper ensemble=127.0.0.1:52614 2024-11-21T08:13:32,978 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T08:13:32,984 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T08:13:33,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:330910x0, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T08:13:33,005 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:330910x0, quorum=127.0.0.1:52614, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T08:13:33,006 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T08:13:33,007 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T08:13:33,008 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33091-0x1015c748eb50003 connected 2024-11-21T08:13:33,010 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33091-0x1015c748eb50003, quorum=127.0.0.1:52614, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T08:13:33,013 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33091-0x1015c748eb50003, quorum=127.0.0.1:52614, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T08:13:33,048 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33091 2024-11-21T08:13:33,049 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33091 2024-11-21T08:13:33,056 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33091 2024-11-21T08:13:33,061 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33091 2024-11-21T08:13:33,062 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33091 2024-11-21T08:13:33,087 DEBUG [M:0;6ed56fb6cbf7:32901 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;6ed56fb6cbf7:32901 2024-11-21T08:13:33,090 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/6ed56fb6cbf7,32901,1732176811582 2024-11-21T08:13:33,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x1015c748eb50003, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T08:13:33,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T08:13:33,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T08:13:33,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T08:13:33,208 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/6ed56fb6cbf7,32901,1732176811582 2024-11-21T08:13:33,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T08:13:33,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T08:13:33,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T08:13:33,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T08:13:33,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T08:13:33,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x1015c748eb50003, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T08:13:33,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x1015c748eb50003, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T08:13:33,340 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T08:13:33,343 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/6ed56fb6cbf7,32901,1732176811582 from backup master directory 2024-11-21T08:13:33,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/6ed56fb6cbf7,32901,1732176811582 2024-11-21T08:13:33,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T08:13:33,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x1015c748eb50003, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T08:13:33,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T08:13:33,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T08:13:33,431 WARN [master/6ed56fb6cbf7:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T08:13:33,431 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=6ed56fb6cbf7,32901,1732176811582 2024-11-21T08:13:33,434 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-21T08:13:33,436 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-21T08:13:33,501 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/hbase.id] with ID: 5fd69682-7f93-4bcf-93cb-dc9c05c86979 2024-11-21T08:13:33,501 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/.tmp/hbase.id 2024-11-21T08:13:33,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741826_1002 (size=42) 2024-11-21T08:13:33,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741826_1002 (size=42) 2024-11-21T08:13:33,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741826_1002 (size=42) 2024-11-21T08:13:33,521 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/.tmp/hbase.id]:[hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/hbase.id] 2024-11-21T08:13:33,573 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T08:13:33,578 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-21T08:13:33,597 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-21T08:13:33,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T08:13:33,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x1015c748eb50003, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T08:13:33,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T08:13:33,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T08:13:33,695 WARN [IPC Server handler 1 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:33,695 WARN [IPC Server handler 1 on default port 45661 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:33,696 WARN [IPC Server handler 1 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:33,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741827_1003 (size=196) 2024-11-21T08:13:33,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741827_1003 (size=196) 2024-11-21T08:13:33,717 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T08:13:33,719 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-21T08:13:33,733 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T08:13:33,736 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T08:13:33,758 WARN [IPC Server handler 2 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:33,759 WARN [IPC Server handler 2 on default port 45661 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:33,759 WARN [IPC Server handler 2 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:33,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741828_1004 (size=1189) 2024-11-21T08:13:33,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741828_1004 (size=1189) 2024-11-21T08:13:33,782 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/MasterData/data/master/store 2024-11-21T08:13:33,796 WARN [IPC Server handler 0 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:33,796 WARN [IPC Server handler 0 on default port 45661 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:33,797 WARN [IPC Server handler 0 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:33,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741829_1005 (size=34) 2024-11-21T08:13:33,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741829_1005 (size=34) 2024-11-21T08:13:33,808 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-21T08:13:33,811 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:33,812 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T08:13:33,812 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T08:13:33,812 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T08:13:33,813 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T08:13:33,813 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T08:13:33,814 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T08:13:33,815 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732176813812Disabling compacts and flushes for region at 1732176813812Disabling writes for close at 1732176813813 (+1 ms)Writing region close event to WAL at 1732176813814 (+1 ms)Closed at 1732176813814 2024-11-21T08:13:33,816 WARN [master/6ed56fb6cbf7:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/MasterData/data/master/store/.initializing 2024-11-21T08:13:33,817 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/MasterData/WALs/6ed56fb6cbf7,32901,1732176811582 2024-11-21T08:13:33,825 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T08:13:33,840 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6ed56fb6cbf7%2C32901%2C1732176811582, suffix=, logDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/MasterData/WALs/6ed56fb6cbf7,32901,1732176811582, archiveDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/MasterData/oldWALs, maxLogs=10 2024-11-21T08:13:33,869 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/MasterData/WALs/6ed56fb6cbf7,32901,1732176811582/6ed56fb6cbf7%2C32901%2C1732176811582.1732176813846, exclude list is [], retry=0 2024-11-21T08:13:33,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-9db7f2dc-3c49-4f47-8420-8e992a2830c6,DISK] 2024-11-21T08:13:33,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33295,DS-7db7df2d-be4b-495c-9b4b-5548750637bb,DISK] 2024-11-21T08:13:33,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34217,DS-f098b1c9-3781-437c-aa42-0db9fe2ba16f,DISK] 2024-11-21T08:13:33,889 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-21T08:13:33,927 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/MasterData/WALs/6ed56fb6cbf7,32901,1732176811582/6ed56fb6cbf7%2C32901%2C1732176811582.1732176813846 2024-11-21T08:13:33,928 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45803:45803),(127.0.0.1/127.0.0.1:46533:46533),(127.0.0.1/127.0.0.1:37401:37401)] 2024-11-21T08:13:33,928 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-21T08:13:33,929 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:33,932 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T08:13:33,933 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T08:13:33,967 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T08:13:33,990 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-21T08:13:33,993 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:33,996 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T08:13:33,996 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T08:13:34,000 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-21T08:13:34,000 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:34,001 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:34,002 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T08:13:34,005 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-21T08:13:34,005 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:34,006 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:34,006 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T08:13:34,009 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-21T08:13:34,009 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:34,010 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:34,011 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T08:13:34,014 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T08:13:34,016 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T08:13:34,021 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T08:13:34,021 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T08:13:34,025 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T08:13:34,029 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T08:13:34,034 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T08:13:34,035 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68227415, jitterRate=0.01666770875453949}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T08:13:34,044 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732176813945Initializing all the Stores at 1732176813947 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732176813948 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176813948Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176813949 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176813949Cleaning up temporary data from old regions at 1732176814021 (+72 ms)Region opened successfully at 1732176814044 (+23 ms) 2024-11-21T08:13:34,045 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-21T08:13:34,080 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24b0a807, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6ed56fb6cbf7/172.17.0.3:0 2024-11-21T08:13:34,108 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-21T08:13:34,118 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T08:13:34,118 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T08:13:34,120 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T08:13:34,122 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-21T08:13:34,127 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-21T08:13:34,127 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-21T08:13:34,153 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-21T08:13:34,161 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-21T08:13:34,204 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-21T08:13:34,208 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-21T08:13:34,210 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-21T08:13:34,214 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-21T08:13:34,217 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-21T08:13:34,220 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-21T08:13:34,225 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-21T08:13:34,226 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-21T08:13:34,235 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-21T08:13:34,254 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-21T08:13:34,267 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-21T08:13:34,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T08:13:34,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T08:13:34,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T08:13:34,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T08:13:34,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x1015c748eb50003, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T08:13:34,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T08:13:34,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T08:13:34,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x1015c748eb50003, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T08:13:34,281 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:34,281 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:34,281 WARN [RedundancyMonitor {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK, ARCHIVE], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:34,282 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) All required storage types are unavailable: unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:34,282 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:34,282 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:34,282 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=6ed56fb6cbf7,32901,1732176811582, sessionid=0x1015c748eb50000, setting cluster-up flag (Was=false) 2024-11-21T08:13:34,283 WARN [RedundancyMonitor {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK, ARCHIVE], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:34,283 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) All required storage types are unavailable: unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:34,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T08:13:34,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T08:13:34,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T08:13:34,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x1015c748eb50003, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T08:13:34,341 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-21T08:13:34,345 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6ed56fb6cbf7,32901,1732176811582 2024-11-21T08:13:34,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T08:13:34,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T08:13:34,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x1015c748eb50003, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T08:13:34,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T08:13:34,404 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-21T08:13:34,408 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6ed56fb6cbf7,32901,1732176811582 2024-11-21T08:13:34,417 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-21T08:13:34,467 INFO [RS:2;6ed56fb6cbf7:33091 {}] regionserver.HRegionServer(746): ClusterId : 5fd69682-7f93-4bcf-93cb-dc9c05c86979 2024-11-21T08:13:34,467 INFO [RS:0;6ed56fb6cbf7:35295 {}] regionserver.HRegionServer(746): ClusterId : 5fd69682-7f93-4bcf-93cb-dc9c05c86979 2024-11-21T08:13:34,467 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.HRegionServer(746): ClusterId : 5fd69682-7f93-4bcf-93cb-dc9c05c86979 2024-11-21T08:13:34,470 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T08:13:34,470 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T08:13:34,470 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T08:13:34,490 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T08:13:34,490 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T08:13:34,490 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T08:13:34,490 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T08:13:34,490 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T08:13:34,490 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T08:13:34,490 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T08:13:34,500 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-21T08:13:34,500 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T08:13:34,500 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T08:13:34,500 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T08:13:34,501 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2392d41b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6ed56fb6cbf7/172.17.0.3:0 2024-11-21T08:13:34,501 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bdbf956, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6ed56fb6cbf7/172.17.0.3:0 2024-11-21T08:13:34,501 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59ad44a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6ed56fb6cbf7/172.17.0.3:0 2024-11-21T08:13:34,508 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-21T08:13:34,518 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;6ed56fb6cbf7:38845 2024-11-21T08:13:34,515 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 6ed56fb6cbf7,32901,1732176811582 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-21T08:13:34,519 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;6ed56fb6cbf7:35295 2024-11-21T08:13:34,519 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;6ed56fb6cbf7:33091 2024-11-21T08:13:34,523 INFO [RS:2;6ed56fb6cbf7:33091 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T08:13:34,523 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T08:13:34,523 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T08:13:34,523 INFO [RS:2;6ed56fb6cbf7:33091 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T08:13:34,523 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T08:13:34,523 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T08:13:34,524 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/6ed56fb6cbf7:0, corePoolSize=5, maxPoolSize=5 2024-11-21T08:13:34,525 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/6ed56fb6cbf7:0, corePoolSize=5, maxPoolSize=5 2024-11-21T08:13:34,525 INFO [RS:0;6ed56fb6cbf7:35295 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T08:13:34,525 INFO [RS:0;6ed56fb6cbf7:35295 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T08:13:34,526 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T08:13:34,526 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/6ed56fb6cbf7:0, corePoolSize=5, maxPoolSize=5 2024-11-21T08:13:34,526 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/6ed56fb6cbf7:0, corePoolSize=5, maxPoolSize=5 2024-11-21T08:13:34,526 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/6ed56fb6cbf7:0, corePoolSize=10, maxPoolSize=10 2024-11-21T08:13:34,527 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,527 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/6ed56fb6cbf7:0, corePoolSize=2, maxPoolSize=2 2024-11-21T08:13:34,527 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,527 INFO [RS:2;6ed56fb6cbf7:33091 {}] regionserver.HRegionServer(2659): reportForDuty to master=6ed56fb6cbf7,32901,1732176811582 with port=33091, startcode=1732176812968 2024-11-21T08:13:34,527 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.HRegionServer(2659): reportForDuty to master=6ed56fb6cbf7,32901,1732176811582 with port=38845, startcode=1732176812866 2024-11-21T08:13:34,528 INFO [RS:0;6ed56fb6cbf7:35295 {}] regionserver.HRegionServer(2659): reportForDuty to master=6ed56fb6cbf7,32901,1732176811582 with port=35295, startcode=1732176812677 2024-11-21T08:13:34,534 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732176844533 2024-11-21T08:13:34,536 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-21T08:13:34,537 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-21T08:13:34,541 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-21T08:13:34,542 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-21T08:13:34,542 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T08:13:34,542 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-21T08:13:34,542 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-21T08:13:34,542 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-21T08:13:34,543 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T08:13:34,543 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T08:13:34,543 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T08:13:34,544 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,548 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:34,549 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-21T08:13:34,549 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-21T08:13:34,550 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-21T08:13:34,550 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-21T08:13:34,553 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-21T08:13:34,554 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-21T08:13:34,557 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/6ed56fb6cbf7:0:becomeActiveMaster-HFileCleaner.large.0-1732176814555,5,FailOnTimeoutGroup] 2024-11-21T08:13:34,559 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/6ed56fb6cbf7:0:becomeActiveMaster-HFileCleaner.small.0-1732176814557,5,FailOnTimeoutGroup] 2024-11-21T08:13:34,559 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,559 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-21T08:13:34,561 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,561 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,609 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42659, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T08:13:34,609 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45987, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T08:13:34,609 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59925, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T08:13:34,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741831_1007 (size=1321) 2024-11-21T08:13:34,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741831_1007 (size=1321) 2024-11-21T08:13:34,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741831_1007 (size=1321) 2024-11-21T08:13:34,614 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-21T08:13:34,615 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:34,617 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:34,620 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] master.ServerManager(517): Registering regionserver=6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:34,624 WARN [IPC Server handler 0 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:34,624 WARN [IPC Server handler 0 on default port 45661 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:34,625 WARN [IPC Server handler 0 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:34,632 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:34,632 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] master.ServerManager(517): Registering regionserver=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:34,637 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:34,638 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] master.ServerManager(517): Registering regionserver=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:34,638 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:34,638 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:34,638 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45661 2024-11-21T08:13:34,638 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45661 2024-11-21T08:13:34,638 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T08:13:34,638 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T08:13:34,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741832_1008 (size=32) 2024-11-21T08:13:34,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741832_1008 (size=32) 2024-11-21T08:13:34,643 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:34,644 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45661 2024-11-21T08:13:34,644 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T08:13:34,644 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:34,648 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T08:13:34,651 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T08:13:34,651 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:34,652 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T08:13:34,653 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T08:13:34,656 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T08:13:34,656 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:34,658 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T08:13:34,658 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T08:13:34,661 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T08:13:34,661 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:34,663 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T08:13:34,663 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T08:13:34,667 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T08:13:34,667 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:34,668 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T08:13:34,668 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T08:13:34,670 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740 2024-11-21T08:13:34,671 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740 2024-11-21T08:13:34,675 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T08:13:34,675 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T08:13:34,677 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T08:13:34,680 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T08:13:34,685 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T08:13:34,686 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59427987, jitterRate=-0.114453986287117}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T08:13:34,691 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732176814645Initializing all the Stores at 1732176814647 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732176814647Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732176814647Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176814647Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732176814647Cleaning up temporary data from old regions at 1732176814675 (+28 ms)Region opened successfully at 1732176814690 (+15 ms) 2024-11-21T08:13:34,691 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T08:13:34,691 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T08:13:34,691 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T08:13:34,691 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T08:13:34,691 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T08:13:34,693 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T08:13:34,693 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732176814691Disabling compacts and flushes for region at 1732176814691Disabling writes for close at 1732176814691Writing region close event to WAL at 1732176814692 (+1 ms)Closed at 1732176814693 (+1 ms) 2024-11-21T08:13:34,696 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T08:13:34,696 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-21T08:13:34,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T08:13:34,714 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T08:13:34,717 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-21T08:13:34,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T08:13:34,776 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] zookeeper.ZKUtil(111): regionserver:33091-0x1015c748eb50003, quorum=127.0.0.1:52614, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:34,776 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] zookeeper.ZKUtil(111): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:34,776 WARN [RS:0;6ed56fb6cbf7:35295 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T08:13:34,776 WARN [RS:2;6ed56fb6cbf7:33091 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T08:13:34,776 INFO [RS:0;6ed56fb6cbf7:35295 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T08:13:34,776 INFO [RS:2;6ed56fb6cbf7:33091 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T08:13:34,777 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:34,777 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:34,777 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] zookeeper.ZKUtil(111): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:34,777 WARN [RS:1;6ed56fb6cbf7:38845 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T08:13:34,777 INFO [RS:1;6ed56fb6cbf7:38845 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T08:13:34,777 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:34,778 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6ed56fb6cbf7,33091,1732176812968] 2024-11-21T08:13:34,778 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6ed56fb6cbf7,38845,1732176812866] 2024-11-21T08:13:34,778 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6ed56fb6cbf7,35295,1732176812677] 2024-11-21T08:13:34,802 INFO [RS:0;6ed56fb6cbf7:35295 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T08:13:34,802 INFO [RS:2;6ed56fb6cbf7:33091 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T08:13:34,802 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T08:13:34,820 INFO [RS:2;6ed56fb6cbf7:33091 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T08:13:34,820 INFO [RS:0;6ed56fb6cbf7:35295 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T08:13:34,820 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T08:13:34,825 INFO [RS:1;6ed56fb6cbf7:38845 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T08:13:34,825 INFO [RS:2;6ed56fb6cbf7:33091 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T08:13:34,825 INFO [RS:0;6ed56fb6cbf7:35295 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T08:13:34,825 INFO [RS:0;6ed56fb6cbf7:35295 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,825 INFO [RS:2;6ed56fb6cbf7:33091 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,825 INFO [RS:1;6ed56fb6cbf7:38845 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,827 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T08:13:34,827 INFO [RS:2;6ed56fb6cbf7:33091 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T08:13:34,827 INFO [RS:0;6ed56fb6cbf7:35295 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T08:13:34,833 INFO [RS:2;6ed56fb6cbf7:33091 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T08:13:34,833 INFO [RS:1;6ed56fb6cbf7:38845 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T08:13:34,833 INFO [RS:0;6ed56fb6cbf7:35295 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T08:13:34,835 INFO [RS:1;6ed56fb6cbf7:38845 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,835 INFO [RS:0;6ed56fb6cbf7:35295 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,835 INFO [RS:2;6ed56fb6cbf7:33091 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,835 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,835 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,835 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,835 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,835 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,835 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,835 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,835 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,835 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,835 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,835 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,835 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,836 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,836 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,836 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,836 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0, corePoolSize=2, maxPoolSize=2 2024-11-21T08:13:34,836 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0, corePoolSize=2, maxPoolSize=2 2024-11-21T08:13:34,836 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0, corePoolSize=2, maxPoolSize=2 2024-11-21T08:13:34,836 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,836 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,836 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,836 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,836 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,836 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,836 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,836 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,836 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,836 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,836 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,836 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,837 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,837 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,837 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,837 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,837 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,837 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6ed56fb6cbf7:0, corePoolSize=1, maxPoolSize=1 2024-11-21T08:13:34,837 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6ed56fb6cbf7:0, corePoolSize=3, maxPoolSize=3 2024-11-21T08:13:34,837 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6ed56fb6cbf7:0, corePoolSize=3, maxPoolSize=3 2024-11-21T08:13:34,837 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6ed56fb6cbf7:0, corePoolSize=3, maxPoolSize=3 2024-11-21T08:13:34,837 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6ed56fb6cbf7:0, corePoolSize=3, maxPoolSize=3 2024-11-21T08:13:34,837 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6ed56fb6cbf7:0, corePoolSize=3, maxPoolSize=3 2024-11-21T08:13:34,837 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6ed56fb6cbf7:0, corePoolSize=3, maxPoolSize=3 2024-11-21T08:13:34,838 INFO [RS:2;6ed56fb6cbf7:33091 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,838 INFO [RS:2;6ed56fb6cbf7:33091 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,839 INFO [RS:2;6ed56fb6cbf7:33091 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,839 INFO [RS:2;6ed56fb6cbf7:33091 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,839 INFO [RS:2;6ed56fb6cbf7:33091 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,839 INFO [RS:2;6ed56fb6cbf7:33091 {}] hbase.ChoreService(168): Chore ScheduledChore name=6ed56fb6cbf7,33091,1732176812968-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T08:13:34,841 INFO [RS:1;6ed56fb6cbf7:38845 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,841 INFO [RS:0;6ed56fb6cbf7:35295 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,841 INFO [RS:1;6ed56fb6cbf7:38845 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,841 INFO [RS:0;6ed56fb6cbf7:35295 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,841 INFO [RS:1;6ed56fb6cbf7:38845 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,841 INFO [RS:0;6ed56fb6cbf7:35295 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,841 INFO [RS:1;6ed56fb6cbf7:38845 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,842 INFO [RS:0;6ed56fb6cbf7:35295 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,842 INFO [RS:1;6ed56fb6cbf7:38845 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,842 INFO [RS:0;6ed56fb6cbf7:35295 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,842 INFO [RS:1;6ed56fb6cbf7:38845 {}] hbase.ChoreService(168): Chore ScheduledChore name=6ed56fb6cbf7,38845,1732176812866-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T08:13:34,842 INFO [RS:0;6ed56fb6cbf7:35295 {}] hbase.ChoreService(168): Chore ScheduledChore name=6ed56fb6cbf7,35295,1732176812677-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T08:13:34,867 INFO [RS:0;6ed56fb6cbf7:35295 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T08:13:34,867 INFO [RS:2;6ed56fb6cbf7:33091 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T08:13:34,867 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T08:13:34,868 WARN [6ed56fb6cbf7:32901 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T08:13:34,869 INFO [RS:1;6ed56fb6cbf7:38845 {}] hbase.ChoreService(168): Chore ScheduledChore name=6ed56fb6cbf7,38845,1732176812866-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,869 INFO [RS:0;6ed56fb6cbf7:35295 {}] hbase.ChoreService(168): Chore ScheduledChore name=6ed56fb6cbf7,35295,1732176812677-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,869 INFO [RS:2;6ed56fb6cbf7:33091 {}] hbase.ChoreService(168): Chore ScheduledChore name=6ed56fb6cbf7,33091,1732176812968-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,870 INFO [RS:0;6ed56fb6cbf7:35295 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,870 INFO [RS:1;6ed56fb6cbf7:38845 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,870 INFO [RS:2;6ed56fb6cbf7:33091 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,870 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.Replication(171): 6ed56fb6cbf7,38845,1732176812866 started 2024-11-21T08:13:34,870 INFO [RS:0;6ed56fb6cbf7:35295 {}] regionserver.Replication(171): 6ed56fb6cbf7,35295,1732176812677 started 2024-11-21T08:13:34,870 INFO [RS:2;6ed56fb6cbf7:33091 {}] regionserver.Replication(171): 6ed56fb6cbf7,33091,1732176812968 started 2024-11-21T08:13:34,886 INFO [RS:1;6ed56fb6cbf7:38845 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,886 INFO [RS:0;6ed56fb6cbf7:35295 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,886 INFO [RS:0;6ed56fb6cbf7:35295 {}] regionserver.HRegionServer(1482): Serving as 6ed56fb6cbf7,35295,1732176812677, RpcServer on 6ed56fb6cbf7/172.17.0.3:35295, sessionid=0x1015c748eb50001 2024-11-21T08:13:34,886 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.HRegionServer(1482): Serving as 6ed56fb6cbf7,38845,1732176812866, RpcServer on 6ed56fb6cbf7/172.17.0.3:38845, sessionid=0x1015c748eb50002 2024-11-21T08:13:34,887 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T08:13:34,887 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T08:13:34,887 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:34,887 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:34,887 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6ed56fb6cbf7,38845,1732176812866' 2024-11-21T08:13:34,887 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6ed56fb6cbf7,35295,1732176812677' 2024-11-21T08:13:34,887 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T08:13:34,888 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T08:13:34,888 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T08:13:34,888 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T08:13:34,889 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T08:13:34,889 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T08:13:34,889 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T08:13:34,889 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T08:13:34,889 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:34,889 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:34,889 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6ed56fb6cbf7,38845,1732176812866' 2024-11-21T08:13:34,889 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6ed56fb6cbf7,35295,1732176812677' 2024-11-21T08:13:34,889 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T08:13:34,889 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T08:13:34,889 INFO [RS:2;6ed56fb6cbf7:33091 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:34,889 INFO [RS:2;6ed56fb6cbf7:33091 {}] regionserver.HRegionServer(1482): Serving as 6ed56fb6cbf7,33091,1732176812968, RpcServer on 6ed56fb6cbf7/172.17.0.3:33091, sessionid=0x1015c748eb50003 2024-11-21T08:13:34,890 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T08:13:34,890 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:34,890 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T08:13:34,890 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6ed56fb6cbf7,33091,1732176812968' 2024-11-21T08:13:34,890 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T08:13:34,890 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T08:13:34,890 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T08:13:34,890 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T08:13:34,890 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T08:13:34,890 INFO [RS:1;6ed56fb6cbf7:38845 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T08:13:34,890 INFO [RS:0;6ed56fb6cbf7:35295 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T08:13:34,890 INFO [RS:0;6ed56fb6cbf7:35295 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T08:13:34,890 INFO [RS:1;6ed56fb6cbf7:38845 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T08:13:34,891 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T08:13:34,891 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T08:13:34,891 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:34,891 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6ed56fb6cbf7,33091,1732176812968' 2024-11-21T08:13:34,891 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T08:13:34,891 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T08:13:34,892 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T08:13:34,892 INFO [RS:2;6ed56fb6cbf7:33091 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T08:13:34,892 INFO [RS:2;6ed56fb6cbf7:33091 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T08:13:35,000 INFO [RS:1;6ed56fb6cbf7:38845 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T08:13:35,000 INFO [RS:2;6ed56fb6cbf7:33091 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T08:13:35,000 INFO [RS:0;6ed56fb6cbf7:35295 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T08:13:35,004 INFO [RS:2;6ed56fb6cbf7:33091 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6ed56fb6cbf7%2C33091%2C1732176812968, suffix=, logDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968, archiveDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/oldWALs, maxLogs=32 2024-11-21T08:13:35,004 INFO [RS:1;6ed56fb6cbf7:38845 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6ed56fb6cbf7%2C38845%2C1732176812866, suffix=, logDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,38845,1732176812866, archiveDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/oldWALs, maxLogs=32 2024-11-21T08:13:35,004 INFO [RS:0;6ed56fb6cbf7:35295 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6ed56fb6cbf7%2C35295%2C1732176812677, suffix=, logDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,35295,1732176812677, archiveDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/oldWALs, maxLogs=32 2024-11-21T08:13:35,022 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968/6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, exclude list is [], retry=0 2024-11-21T08:13:35,022 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,35295,1732176812677/6ed56fb6cbf7%2C35295%2C1732176812677.1732176815010, exclude list is [], retry=0 2024-11-21T08:13:35,024 WARN [IPC Server handler 4 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:35,025 WARN [IPC Server handler 3 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:35,025 WARN [IPC Server handler 4 on default port 45661 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:35,025 WARN [IPC Server handler 3 on default port 45661 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:35,025 WARN [IPC Server handler 4 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:35,025 WARN [IPC Server handler 3 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:35,027 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33295,DS-7db7df2d-be4b-495c-9b4b-5548750637bb,DISK] 2024-11-21T08:13:35,027 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33295,DS-7db7df2d-be4b-495c-9b4b-5548750637bb,DISK] 2024-11-21T08:13:35,027 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34217,DS-f098b1c9-3781-437c-aa42-0db9fe2ba16f,DISK] 2024-11-21T08:13:35,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34217,DS-f098b1c9-3781-437c-aa42-0db9fe2ba16f,DISK] 2024-11-21T08:13:35,028 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,38845,1732176812866/6ed56fb6cbf7%2C38845%2C1732176812866.1732176815010, exclude list is [], retry=0 2024-11-21T08:13:35,031 WARN [IPC Server handler 3 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:35,031 WARN [IPC Server handler 3 on default port 45661 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:35,031 WARN [IPC Server handler 3 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:35,032 INFO [RS:2;6ed56fb6cbf7:33091 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968/6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010 2024-11-21T08:13:35,033 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37401:37401),(127.0.0.1/127.0.0.1:46533:46533)] 2024-11-21T08:13:35,034 INFO [RS:0;6ed56fb6cbf7:35295 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,35295,1732176812677/6ed56fb6cbf7%2C35295%2C1732176812677.1732176815010 2024-11-21T08:13:35,036 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46533:46533),(127.0.0.1/127.0.0.1:37401:37401)] 2024-11-21T08:13:35,036 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34217,DS-f098b1c9-3781-437c-aa42-0db9fe2ba16f,DISK] 2024-11-21T08:13:35,036 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33295,DS-7db7df2d-be4b-495c-9b4b-5548750637bb,DISK] 2024-11-21T08:13:35,042 INFO [RS:1;6ed56fb6cbf7:38845 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,38845,1732176812866/6ed56fb6cbf7%2C38845%2C1732176812866.1732176815010 2024-11-21T08:13:35,042 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37401:37401),(127.0.0.1/127.0.0.1:46533:46533)] 2024-11-21T08:13:35,123 DEBUG [6ed56fb6cbf7:32901 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-21T08:13:35,136 DEBUG [6ed56fb6cbf7:32901 {}] balancer.BalancerClusterState(204): Hosts are {6ed56fb6cbf7=0} racks are {/default-rack=0} 2024-11-21T08:13:35,142 DEBUG [6ed56fb6cbf7:32901 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T08:13:35,142 DEBUG [6ed56fb6cbf7:32901 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T08:13:35,142 DEBUG [6ed56fb6cbf7:32901 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-21T08:13:35,142 DEBUG [6ed56fb6cbf7:32901 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T08:13:35,142 DEBUG [6ed56fb6cbf7:32901 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T08:13:35,142 DEBUG [6ed56fb6cbf7:32901 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-21T08:13:35,142 INFO [6ed56fb6cbf7:32901 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T08:13:35,142 INFO [6ed56fb6cbf7:32901 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T08:13:35,142 INFO [6ed56fb6cbf7:32901 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-21T08:13:35,142 DEBUG [6ed56fb6cbf7:32901 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T08:13:35,149 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:35,155 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ed56fb6cbf7,33091,1732176812968, state=OPENING 2024-11-21T08:13:35,204 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-21T08:13:35,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T08:13:35,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T08:13:35,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T08:13:35,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x1015c748eb50003, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T08:13:35,217 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:35,217 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:35,217 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:35,217 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:35,218 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T08:13:35,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,33091,1732176812968}] 2024-11-21T08:13:35,395 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T08:13:35,397 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55217, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T08:13:35,411 INFO [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T08:13:35,412 INFO [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T08:13:35,412 INFO [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T08:13:35,416 INFO [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6ed56fb6cbf7%2C33091%2C1732176812968.meta, suffix=.meta, logDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968, archiveDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/oldWALs, maxLogs=32 2024-11-21T08:13:35,434 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968/6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, exclude list is [], retry=0 2024-11-21T08:13:35,437 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-9db7f2dc-3c49-4f47-8420-8e992a2830c6,DISK] 2024-11-21T08:13:35,437 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33295,DS-7db7df2d-be4b-495c-9b4b-5548750637bb,DISK] 2024-11-21T08:13:35,438 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34217,DS-f098b1c9-3781-437c-aa42-0db9fe2ba16f,DISK] 2024-11-21T08:13:35,443 INFO [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968/6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta 2024-11-21T08:13:35,444 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45803:45803),(127.0.0.1/127.0.0.1:46533:46533),(127.0.0.1/127.0.0.1:37401:37401)] 2024-11-21T08:13:35,444 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T08:13:35,446 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T08:13:35,448 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T08:13:35,452 INFO [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T08:13:35,458 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T08:13:35,458 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:35,459 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T08:13:35,459 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T08:13:35,464 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T08:13:35,467 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T08:13:35,467 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:35,468 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T08:13:35,468 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T08:13:35,470 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T08:13:35,470 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:35,471 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T08:13:35,471 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T08:13:35,473 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T08:13:35,473 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:35,474 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T08:13:35,475 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T08:13:35,476 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T08:13:35,476 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:35,478 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T08:13:35,478 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T08:13:35,480 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740 2024-11-21T08:13:35,483 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740 2024-11-21T08:13:35,486 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T08:13:35,486 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T08:13:35,488 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T08:13:35,491 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T08:13:35,494 INFO [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58991164, jitterRate=-0.12096315622329712}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T08:13:35,494 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T08:13:35,496 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732176815460Writing region info on filesystem at 1732176815460Initializing all the Stores at 1732176815462 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732176815462Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732176815464 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176815464Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732176815464Cleaning up temporary data from old regions at 1732176815486 (+22 ms)Running coprocessor post-open hooks at 1732176815494 (+8 ms)Region opened successfully at 1732176815496 (+2 ms) 2024-11-21T08:13:35,505 INFO [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732176815386 2024-11-21T08:13:35,519 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T08:13:35,520 INFO [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T08:13:35,523 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:35,527 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ed56fb6cbf7,33091,1732176812968, state=OPEN 2024-11-21T08:13:35,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:35,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:35,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:35,607 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:35,607 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:35,607 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:35,607 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:35,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x1015c748eb50003, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:35,612 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:35,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-21T08:13:35,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,33091,1732176812968 in 388 msec 2024-11-21T08:13:35,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-21T08:13:35,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 918 msec 2024-11-21T08:13:35,643 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T08:13:35,643 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-21T08:13:35,692 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T08:13:35,695 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ed56fb6cbf7,33091,1732176812968, seqNum=-1] 2024-11-21T08:13:35,727 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T08:13:35,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36945, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T08:13:35,797 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3310 sec 2024-11-21T08:13:35,797 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732176815797, completionTime=-1 2024-11-21T08:13:35,801 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-21T08:13:35,801 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T08:13:35,864 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-21T08:13:35,864 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732176875864 2024-11-21T08:13:35,864 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732176935864 2024-11-21T08:13:35,864 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 62 msec 2024-11-21T08:13:35,867 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-21T08:13:35,882 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ed56fb6cbf7,32901,1732176811582-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:35,883 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ed56fb6cbf7,32901,1732176811582-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:35,883 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ed56fb6cbf7,32901,1732176811582-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:35,885 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-6ed56fb6cbf7:32901, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:35,886 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:35,889 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:35,902 DEBUG [master/6ed56fb6cbf7:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-21T08:13:35,929 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.498sec 2024-11-21T08:13:35,931 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-21T08:13:35,933 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-21T08:13:35,938 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-21T08:13:35,940 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-21T08:13:35,940 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-21T08:13:35,941 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ed56fb6cbf7,32901,1732176811582-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T08:13:35,942 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ed56fb6cbf7,32901,1732176811582-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-21T08:13:35,948 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-21T08:13:35,949 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-21T08:13:35,949 INFO [master/6ed56fb6cbf7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ed56fb6cbf7,32901,1732176811582-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T08:13:36,000 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9330f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T08:13:36,006 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-21T08:13:36,006 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-21T08:13:36,014 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6ed56fb6cbf7,32901,-1 for getting cluster id 2024-11-21T08:13:36,018 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T08:13:36,032 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5fd69682-7f93-4bcf-93cb-dc9c05c86979' 2024-11-21T08:13:36,037 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T08:13:36,037 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5fd69682-7f93-4bcf-93cb-dc9c05c86979" 2024-11-21T08:13:36,038 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16bf1c85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T08:13:36,038 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6ed56fb6cbf7,32901,-1] 2024-11-21T08:13:36,042 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T08:13:36,046 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T08:13:36,048 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T08:13:36,052 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6982d266, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T08:13:36,053 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T08:13:36,063 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ed56fb6cbf7,33091,1732176812968, seqNum=-1] 2024-11-21T08:13:36,063 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T08:13:36,067 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T08:13:36,090 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=6ed56fb6cbf7,32901,1732176811582 2024-11-21T08:13:36,095 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T08:13:36,102 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 6ed56fb6cbf7,32901,1732176811582 2024-11-21T08:13:36,104 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6e25a549 2024-11-21T08:13:36,105 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T08:13:36,146 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T08:13:36,154 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T08:13:36,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-21T08:13:36,173 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T08:13:36,175 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-21T08:13:36,176 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:36,186 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T08:13:36,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T08:13:36,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741837_1013 (size=358) 2024-11-21T08:13:36,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741837_1013 (size=358) 2024-11-21T08:13:36,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741837_1013 (size=358) 2024-11-21T08:13:36,223 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fd0a67dc199543d2675c547a7cb41702, NAME => 'test,,1732176816148.fd0a67dc199543d2675c547a7cb41702.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:36,225 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => f38fd8b8defc002b7cb3de558717f27c, NAME => 'test,ccc,1732176816148.f38fd8b8defc002b7cb3de558717f27c.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:36,227 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 3773623806176f3d7f8efb3efe1b7937, NAME => 'test,bbb,1732176816148.3773623806176f3d7f8efb3efe1b7937.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:36,249 WARN [IPC Server handler 0 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:36,249 WARN [IPC Server handler 4 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:36,249 WARN [IPC Server handler 0 on default port 45661 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:36,249 WARN [IPC Server handler 4 on default port 45661 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:36,249 WARN [IPC Server handler 0 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:36,249 WARN [IPC Server handler 4 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:36,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741840_1016 (size=45) 2024-11-21T08:13:36,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741840_1016 (size=45) 2024-11-21T08:13:36,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741838_1014 (size=42) 2024-11-21T08:13:36,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741838_1014 (size=42) 2024-11-21T08:13:36,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741838_1014 (size=42) 2024-11-21T08:13:36,284 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1732176816148.fd0a67dc199543d2675c547a7cb41702.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:36,285 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing fd0a67dc199543d2675c547a7cb41702, disabling compactions & flushes 2024-11-21T08:13:36,285 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1732176816148.fd0a67dc199543d2675c547a7cb41702. 2024-11-21T08:13:36,285 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732176816148.fd0a67dc199543d2675c547a7cb41702. 2024-11-21T08:13:36,285 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1732176816148.fd0a67dc199543d2675c547a7cb41702. after waiting 0 ms 2024-11-21T08:13:36,285 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1732176816148.fd0a67dc199543d2675c547a7cb41702. 2024-11-21T08:13:36,285 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1732176816148.fd0a67dc199543d2675c547a7cb41702. 2024-11-21T08:13:36,285 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for fd0a67dc199543d2675c547a7cb41702: Waiting for close lock at 1732176816284Disabling compacts and flushes for region at 1732176816284Disabling writes for close at 1732176816285 (+1 ms)Writing region close event to WAL at 1732176816285Closed at 1732176816285 2024-11-21T08:13:36,285 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4f43187c46588bff73d01ba7eb26a6f8, NAME => 'test,ddd,1732176816148.4f43187c46588bff73d01ba7eb26a6f8.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:36,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741839_1015 (size=45) 2024-11-21T08:13:36,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741839_1015 (size=45) 2024-11-21T08:13:36,288 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1732176816148.f38fd8b8defc002b7cb3de558717f27c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:36,288 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing f38fd8b8defc002b7cb3de558717f27c, disabling compactions & flushes 2024-11-21T08:13:36,288 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1732176816148.f38fd8b8defc002b7cb3de558717f27c. 2024-11-21T08:13:36,288 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732176816148.f38fd8b8defc002b7cb3de558717f27c. 2024-11-21T08:13:36,288 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732176816148.f38fd8b8defc002b7cb3de558717f27c. after waiting 0 ms 2024-11-21T08:13:36,288 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732176816148.f38fd8b8defc002b7cb3de558717f27c. 2024-11-21T08:13:36,288 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1732176816148.f38fd8b8defc002b7cb3de558717f27c. 2024-11-21T08:13:36,289 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for f38fd8b8defc002b7cb3de558717f27c: Waiting for close lock at 1732176816288Disabling compacts and flushes for region at 1732176816288Disabling writes for close at 1732176816288Writing region close event to WAL at 1732176816288Closed at 1732176816288 2024-11-21T08:13:36,289 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => cd29f056dad8c6c37d0979434891505d, NAME => 'test,eee,1732176816148.cd29f056dad8c6c37d0979434891505d.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:36,291 WARN [IPC Server handler 1 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:36,291 WARN [IPC Server handler 1 on default port 45661 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:36,291 WARN [IPC Server handler 1 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:36,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T08:13:36,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741841_1017 (size=45) 2024-11-21T08:13:36,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741841_1017 (size=45) 2024-11-21T08:13:36,314 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ddd,1732176816148.4f43187c46588bff73d01ba7eb26a6f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:36,314 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 4f43187c46588bff73d01ba7eb26a6f8, disabling compactions & flushes 2024-11-21T08:13:36,315 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ddd,1732176816148.4f43187c46588bff73d01ba7eb26a6f8. 2024-11-21T08:13:36,315 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732176816148.4f43187c46588bff73d01ba7eb26a6f8. 2024-11-21T08:13:36,315 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732176816148.4f43187c46588bff73d01ba7eb26a6f8. after waiting 0 ms 2024-11-21T08:13:36,315 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732176816148.4f43187c46588bff73d01ba7eb26a6f8. 2024-11-21T08:13:36,315 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ddd,1732176816148.4f43187c46588bff73d01ba7eb26a6f8. 2024-11-21T08:13:36,315 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4f43187c46588bff73d01ba7eb26a6f8: Waiting for close lock at 1732176816314Disabling compacts and flushes for region at 1732176816314Disabling writes for close at 1732176816315 (+1 ms)Writing region close event to WAL at 1732176816315Closed at 1732176816315 2024-11-21T08:13:36,316 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 829529fdccf9853d96113fa67713ffbc, NAME => 'test,fff,1732176816148.829529fdccf9853d96113fa67713ffbc.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:36,326 WARN [IPC Server handler 0 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:36,326 WARN [IPC Server handler 0 on default port 45661 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:36,326 WARN [IPC Server handler 0 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:36,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741842_1018 (size=45) 2024-11-21T08:13:36,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741842_1018 (size=45) 2024-11-21T08:13:36,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741842_1018 (size=45) 2024-11-21T08:13:36,336 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,eee,1732176816148.cd29f056dad8c6c37d0979434891505d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:36,336 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing cd29f056dad8c6c37d0979434891505d, disabling compactions & flushes 2024-11-21T08:13:36,336 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,eee,1732176816148.cd29f056dad8c6c37d0979434891505d. 2024-11-21T08:13:36,336 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732176816148.cd29f056dad8c6c37d0979434891505d. 2024-11-21T08:13:36,337 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732176816148.cd29f056dad8c6c37d0979434891505d. after waiting 0 ms 2024-11-21T08:13:36,337 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732176816148.cd29f056dad8c6c37d0979434891505d. 2024-11-21T08:13:36,337 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,eee,1732176816148.cd29f056dad8c6c37d0979434891505d. 2024-11-21T08:13:36,337 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for cd29f056dad8c6c37d0979434891505d: Waiting for close lock at 1732176816336Disabling compacts and flushes for region at 1732176816336Disabling writes for close at 1732176816337 (+1 ms)Writing region close event to WAL at 1732176816337Closed at 1732176816337 2024-11-21T08:13:36,337 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => ee7ef14c3df8915e550cfda528563489, NAME => 'test,ggg,1732176816148.ee7ef14c3df8915e550cfda528563489.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:36,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741843_1019 (size=45) 2024-11-21T08:13:36,344 WARN [IPC Server handler 3 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:36,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741843_1019 (size=45) 2024-11-21T08:13:36,344 WARN [IPC Server handler 3 on default port 45661 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:36,345 WARN [IPC Server handler 3 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:36,348 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,fff,1732176816148.829529fdccf9853d96113fa67713ffbc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:36,348 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 829529fdccf9853d96113fa67713ffbc, disabling compactions & flushes 2024-11-21T08:13:36,348 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,fff,1732176816148.829529fdccf9853d96113fa67713ffbc. 2024-11-21T08:13:36,348 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732176816148.829529fdccf9853d96113fa67713ffbc. 2024-11-21T08:13:36,349 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732176816148.829529fdccf9853d96113fa67713ffbc. after waiting 0 ms 2024-11-21T08:13:36,349 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732176816148.829529fdccf9853d96113fa67713ffbc. 2024-11-21T08:13:36,349 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,fff,1732176816148.829529fdccf9853d96113fa67713ffbc. 2024-11-21T08:13:36,349 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 829529fdccf9853d96113fa67713ffbc: Waiting for close lock at 1732176816348Disabling compacts and flushes for region at 1732176816348Disabling writes for close at 1732176816349 (+1 ms)Writing region close event to WAL at 1732176816349Closed at 1732176816349 2024-11-21T08:13:36,350 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9eb0364d53b384c3e59ecc15348e6c3c, NAME => 'test,hhh,1732176816148.9eb0364d53b384c3e59ecc15348e6c3c.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:36,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741844_1020 (size=45) 2024-11-21T08:13:36,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741844_1020 (size=45) 2024-11-21T08:13:36,365 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ggg,1732176816148.ee7ef14c3df8915e550cfda528563489.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:36,365 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing ee7ef14c3df8915e550cfda528563489, disabling compactions & flushes 2024-11-21T08:13:36,365 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ggg,1732176816148.ee7ef14c3df8915e550cfda528563489. 2024-11-21T08:13:36,365 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732176816148.ee7ef14c3df8915e550cfda528563489. 2024-11-21T08:13:36,365 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732176816148.ee7ef14c3df8915e550cfda528563489. after waiting 0 ms 2024-11-21T08:13:36,365 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732176816148.ee7ef14c3df8915e550cfda528563489. 2024-11-21T08:13:36,366 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ggg,1732176816148.ee7ef14c3df8915e550cfda528563489. 2024-11-21T08:13:36,366 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for ee7ef14c3df8915e550cfda528563489: Waiting for close lock at 1732176816365Disabling compacts and flushes for region at 1732176816365Disabling writes for close at 1732176816365Writing region close event to WAL at 1732176816366 (+1 ms)Closed at 1732176816366 2024-11-21T08:13:36,367 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => fd97d8f813bdfcba3626cc4950c7c1e1, NAME => 'test,iii,1732176816148.fd97d8f813bdfcba3626cc4950c7c1e1.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:36,372 WARN [IPC Server handler 2 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:36,372 WARN [IPC Server handler 2 on default port 45661 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:36,372 WARN [IPC Server handler 2 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:36,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741845_1021 (size=45) 2024-11-21T08:13:36,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741845_1021 (size=45) 2024-11-21T08:13:36,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741845_1021 (size=45) 2024-11-21T08:13:36,377 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,hhh,1732176816148.9eb0364d53b384c3e59ecc15348e6c3c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:36,377 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 9eb0364d53b384c3e59ecc15348e6c3c, disabling compactions & flushes 2024-11-21T08:13:36,377 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,hhh,1732176816148.9eb0364d53b384c3e59ecc15348e6c3c. 2024-11-21T08:13:36,377 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732176816148.9eb0364d53b384c3e59ecc15348e6c3c. 2024-11-21T08:13:36,377 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732176816148.9eb0364d53b384c3e59ecc15348e6c3c. after waiting 0 ms 2024-11-21T08:13:36,377 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732176816148.9eb0364d53b384c3e59ecc15348e6c3c. 2024-11-21T08:13:36,377 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,hhh,1732176816148.9eb0364d53b384c3e59ecc15348e6c3c. 2024-11-21T08:13:36,377 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9eb0364d53b384c3e59ecc15348e6c3c: Waiting for close lock at 1732176816377Disabling compacts and flushes for region at 1732176816377Disabling writes for close at 1732176816377Writing region close event to WAL at 1732176816377Closed at 1732176816377 2024-11-21T08:13:36,378 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5fb158c8201cfbc5babdbfc33ee45ab1, NAME => 'test,jjj,1732176816148.5fb158c8201cfbc5babdbfc33ee45ab1.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:36,385 WARN [IPC Server handler 3 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:36,385 WARN [IPC Server handler 3 on default port 45661 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:36,385 WARN [IPC Server handler 3 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:36,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741846_1022 (size=45) 2024-11-21T08:13:36,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741846_1022 (size=45) 2024-11-21T08:13:36,390 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,iii,1732176816148.fd97d8f813bdfcba3626cc4950c7c1e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:36,390 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing fd97d8f813bdfcba3626cc4950c7c1e1, disabling compactions & flushes 2024-11-21T08:13:36,390 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,iii,1732176816148.fd97d8f813bdfcba3626cc4950c7c1e1. 2024-11-21T08:13:36,390 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732176816148.fd97d8f813bdfcba3626cc4950c7c1e1. 2024-11-21T08:13:36,390 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732176816148.fd97d8f813bdfcba3626cc4950c7c1e1. after waiting 0 ms 2024-11-21T08:13:36,390 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732176816148.fd97d8f813bdfcba3626cc4950c7c1e1. 2024-11-21T08:13:36,390 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,iii,1732176816148.fd97d8f813bdfcba3626cc4950c7c1e1. 2024-11-21T08:13:36,391 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for fd97d8f813bdfcba3626cc4950c7c1e1: Waiting for close lock at 1732176816390Disabling compacts and flushes for region at 1732176816390Disabling writes for close at 1732176816390Writing region close event to WAL at 1732176816390Closed at 1732176816390 2024-11-21T08:13:36,391 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => dbfd5210cf46b2a246edeeebd244b662, NAME => 'test,kkk,1732176816148.dbfd5210cf46b2a246edeeebd244b662.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:36,397 WARN [IPC Server handler 2 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:36,397 WARN [IPC Server handler 2 on default port 45661 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:36,398 WARN [IPC Server handler 2 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:36,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741847_1023 (size=45) 2024-11-21T08:13:36,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741847_1023 (size=45) 2024-11-21T08:13:36,403 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,jjj,1732176816148.5fb158c8201cfbc5babdbfc33ee45ab1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:36,403 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 5fb158c8201cfbc5babdbfc33ee45ab1, disabling compactions & flushes 2024-11-21T08:13:36,403 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,jjj,1732176816148.5fb158c8201cfbc5babdbfc33ee45ab1. 2024-11-21T08:13:36,404 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732176816148.5fb158c8201cfbc5babdbfc33ee45ab1. 2024-11-21T08:13:36,404 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732176816148.5fb158c8201cfbc5babdbfc33ee45ab1. after waiting 0 ms 2024-11-21T08:13:36,404 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732176816148.5fb158c8201cfbc5babdbfc33ee45ab1. 2024-11-21T08:13:36,404 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,jjj,1732176816148.5fb158c8201cfbc5babdbfc33ee45ab1. 2024-11-21T08:13:36,404 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5fb158c8201cfbc5babdbfc33ee45ab1: Waiting for close lock at 1732176816403Disabling compacts and flushes for region at 1732176816403Disabling writes for close at 1732176816404 (+1 ms)Writing region close event to WAL at 1732176816404Closed at 1732176816404 2024-11-21T08:13:36,405 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 94c604fd0d435e28ac643b49d6cbee14, NAME => 'test,lll,1732176816148.94c604fd0d435e28ac643b49d6cbee14.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:36,409 WARN [IPC Server handler 3 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:36,410 WARN [IPC Server handler 3 on default port 45661 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:36,410 WARN [IPC Server handler 3 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:36,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741848_1024 (size=45) 2024-11-21T08:13:36,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741848_1024 (size=45) 2024-11-21T08:13:36,412 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,kkk,1732176816148.dbfd5210cf46b2a246edeeebd244b662.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:36,412 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing dbfd5210cf46b2a246edeeebd244b662, disabling compactions & flushes 2024-11-21T08:13:36,412 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,kkk,1732176816148.dbfd5210cf46b2a246edeeebd244b662. 2024-11-21T08:13:36,412 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732176816148.dbfd5210cf46b2a246edeeebd244b662. 2024-11-21T08:13:36,412 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732176816148.dbfd5210cf46b2a246edeeebd244b662. after waiting 0 ms 2024-11-21T08:13:36,413 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732176816148.dbfd5210cf46b2a246edeeebd244b662. 2024-11-21T08:13:36,413 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,kkk,1732176816148.dbfd5210cf46b2a246edeeebd244b662. 2024-11-21T08:13:36,413 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for dbfd5210cf46b2a246edeeebd244b662: Waiting for close lock at 1732176816412Disabling compacts and flushes for region at 1732176816412Disabling writes for close at 1732176816412Writing region close event to WAL at 1732176816413 (+1 ms)Closed at 1732176816413 2024-11-21T08:13:36,414 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => d72691fc99dffac743de3d6bcb0f84ed, NAME => 'test,mmm,1732176816148.d72691fc99dffac743de3d6bcb0f84ed.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:36,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741849_1025 (size=45) 2024-11-21T08:13:36,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741849_1025 (size=45) 2024-11-21T08:13:36,419 WARN [IPC Server handler 2 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:36,419 WARN [IPC Server handler 2 on default port 45661 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:36,419 WARN [IPC Server handler 2 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:36,421 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,lll,1732176816148.94c604fd0d435e28ac643b49d6cbee14.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:36,421 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 94c604fd0d435e28ac643b49d6cbee14, disabling compactions & flushes 2024-11-21T08:13:36,421 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,lll,1732176816148.94c604fd0d435e28ac643b49d6cbee14. 2024-11-21T08:13:36,421 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732176816148.94c604fd0d435e28ac643b49d6cbee14. 2024-11-21T08:13:36,421 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732176816148.94c604fd0d435e28ac643b49d6cbee14. after waiting 0 ms 2024-11-21T08:13:36,421 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732176816148.94c604fd0d435e28ac643b49d6cbee14. 2024-11-21T08:13:36,421 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,lll,1732176816148.94c604fd0d435e28ac643b49d6cbee14. 2024-11-21T08:13:36,422 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 94c604fd0d435e28ac643b49d6cbee14: Waiting for close lock at 1732176816421Disabling compacts and flushes for region at 1732176816421Disabling writes for close at 1732176816421Writing region close event to WAL at 1732176816421Closed at 1732176816421 2024-11-21T08:13:36,423 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 99e7b3565f08f10e8c368072e9c21c09, NAME => 'test,nnn,1732176816148.99e7b3565f08f10e8c368072e9c21c09.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:36,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741850_1026 (size=45) 2024-11-21T08:13:36,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741850_1026 (size=45) 2024-11-21T08:13:36,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741851_1027 (size=45) 2024-11-21T08:13:36,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741851_1027 (size=45) 2024-11-21T08:13:36,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741851_1027 (size=45) 2024-11-21T08:13:36,441 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,nnn,1732176816148.99e7b3565f08f10e8c368072e9c21c09.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:36,441 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 99e7b3565f08f10e8c368072e9c21c09, disabling compactions & flushes 2024-11-21T08:13:36,441 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,nnn,1732176816148.99e7b3565f08f10e8c368072e9c21c09. 2024-11-21T08:13:36,441 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732176816148.99e7b3565f08f10e8c368072e9c21c09. 2024-11-21T08:13:36,441 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732176816148.99e7b3565f08f10e8c368072e9c21c09. after waiting 0 ms 2024-11-21T08:13:36,441 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732176816148.99e7b3565f08f10e8c368072e9c21c09. 2024-11-21T08:13:36,441 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,nnn,1732176816148.99e7b3565f08f10e8c368072e9c21c09. 2024-11-21T08:13:36,441 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 99e7b3565f08f10e8c368072e9c21c09: Waiting for close lock at 1732176816441Disabling compacts and flushes for region at 1732176816441Disabling writes for close at 1732176816441Writing region close event to WAL at 1732176816441Closed at 1732176816441 2024-11-21T08:13:36,442 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 477e65b3290a1e6e03e5eb59f7aa956d, NAME => 'test,ooo,1732176816148.477e65b3290a1e6e03e5eb59f7aa956d.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:36,447 WARN [IPC Server handler 3 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:36,447 WARN [IPC Server handler 3 on default port 45661 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:36,447 WARN [IPC Server handler 3 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:36,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741852_1028 (size=45) 2024-11-21T08:13:36,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741852_1028 (size=45) 2024-11-21T08:13:36,456 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ooo,1732176816148.477e65b3290a1e6e03e5eb59f7aa956d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:36,456 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 477e65b3290a1e6e03e5eb59f7aa956d, disabling compactions & flushes 2024-11-21T08:13:36,456 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ooo,1732176816148.477e65b3290a1e6e03e5eb59f7aa956d. 2024-11-21T08:13:36,456 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732176816148.477e65b3290a1e6e03e5eb59f7aa956d. 2024-11-21T08:13:36,456 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732176816148.477e65b3290a1e6e03e5eb59f7aa956d. after waiting 0 ms 2024-11-21T08:13:36,456 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732176816148.477e65b3290a1e6e03e5eb59f7aa956d. 2024-11-21T08:13:36,456 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ooo,1732176816148.477e65b3290a1e6e03e5eb59f7aa956d. 2024-11-21T08:13:36,456 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 477e65b3290a1e6e03e5eb59f7aa956d: Waiting for close lock at 1732176816456Disabling compacts and flushes for region at 1732176816456Disabling writes for close at 1732176816456Writing region close event to WAL at 1732176816456Closed at 1732176816456 2024-11-21T08:13:36,457 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5017ae3acb74fe8c7d0b9c6d9a2cc402, NAME => 'test,ppp,1732176816148.5017ae3acb74fe8c7d0b9c6d9a2cc402.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:36,461 WARN [IPC Server handler 0 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:36,461 WARN [IPC Server handler 0 on default port 45661 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:36,461 WARN [IPC Server handler 0 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:36,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741853_1029 (size=45) 2024-11-21T08:13:36,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741853_1029 (size=45) 2024-11-21T08:13:36,468 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ppp,1732176816148.5017ae3acb74fe8c7d0b9c6d9a2cc402.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:36,468 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 5017ae3acb74fe8c7d0b9c6d9a2cc402, disabling compactions & flushes 2024-11-21T08:13:36,468 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ppp,1732176816148.5017ae3acb74fe8c7d0b9c6d9a2cc402. 2024-11-21T08:13:36,468 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732176816148.5017ae3acb74fe8c7d0b9c6d9a2cc402. 2024-11-21T08:13:36,469 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732176816148.5017ae3acb74fe8c7d0b9c6d9a2cc402. after waiting 0 ms 2024-11-21T08:13:36,469 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732176816148.5017ae3acb74fe8c7d0b9c6d9a2cc402. 2024-11-21T08:13:36,469 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ppp,1732176816148.5017ae3acb74fe8c7d0b9c6d9a2cc402. 2024-11-21T08:13:36,469 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5017ae3acb74fe8c7d0b9c6d9a2cc402: Waiting for close lock at 1732176816468Disabling compacts and flushes for region at 1732176816468Disabling writes for close at 1732176816469 (+1 ms)Writing region close event to WAL at 1732176816469Closed at 1732176816469 2024-11-21T08:13:36,469 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2d5b1f002057972e76beae8f19094030, NAME => 'test,qqq,1732176816148.2d5b1f002057972e76beae8f19094030.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:36,473 WARN [IPC Server handler 2 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:36,473 WARN [IPC Server handler 2 on default port 45661 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:36,473 WARN [IPC Server handler 2 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:36,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741854_1030 (size=45) 2024-11-21T08:13:36,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741854_1030 (size=45) 2024-11-21T08:13:36,480 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,qqq,1732176816148.2d5b1f002057972e76beae8f19094030.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:36,480 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 2d5b1f002057972e76beae8f19094030, disabling compactions & flushes 2024-11-21T08:13:36,481 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,qqq,1732176816148.2d5b1f002057972e76beae8f19094030. 2024-11-21T08:13:36,481 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732176816148.2d5b1f002057972e76beae8f19094030. 2024-11-21T08:13:36,481 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732176816148.2d5b1f002057972e76beae8f19094030. after waiting 0 ms 2024-11-21T08:13:36,481 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732176816148.2d5b1f002057972e76beae8f19094030. 2024-11-21T08:13:36,481 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,qqq,1732176816148.2d5b1f002057972e76beae8f19094030. 2024-11-21T08:13:36,481 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2d5b1f002057972e76beae8f19094030: Waiting for close lock at 1732176816480Disabling compacts and flushes for region at 1732176816480Disabling writes for close at 1732176816481 (+1 ms)Writing region close event to WAL at 1732176816481Closed at 1732176816481 2024-11-21T08:13:36,482 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f55c025b8995c15117005602a775de64, NAME => 'test,rrr,1732176816148.f55c025b8995c15117005602a775de64.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:36,486 WARN [IPC Server handler 1 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:36,486 WARN [IPC Server handler 1 on default port 45661 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:36,486 WARN [IPC Server handler 1 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:36,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741855_1031 (size=45) 2024-11-21T08:13:36,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741855_1031 (size=45) 2024-11-21T08:13:36,494 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,rrr,1732176816148.f55c025b8995c15117005602a775de64.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:36,494 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing f55c025b8995c15117005602a775de64, disabling compactions & flushes 2024-11-21T08:13:36,494 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,rrr,1732176816148.f55c025b8995c15117005602a775de64. 2024-11-21T08:13:36,494 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732176816148.f55c025b8995c15117005602a775de64. 2024-11-21T08:13:36,494 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732176816148.f55c025b8995c15117005602a775de64. after waiting 0 ms 2024-11-21T08:13:36,495 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732176816148.f55c025b8995c15117005602a775de64. 2024-11-21T08:13:36,495 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,rrr,1732176816148.f55c025b8995c15117005602a775de64. 2024-11-21T08:13:36,495 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for f55c025b8995c15117005602a775de64: Waiting for close lock at 1732176816494Disabling compacts and flushes for region at 1732176816494Disabling writes for close at 1732176816495 (+1 ms)Writing region close event to WAL at 1732176816495Closed at 1732176816495 2024-11-21T08:13:36,495 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4ed3111e35326fc2c5455ae2475ab1b5, NAME => 'test,sss,1732176816148.4ed3111e35326fc2c5455ae2475ab1b5.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:36,499 WARN [IPC Server handler 2 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:36,499 WARN [IPC Server handler 2 on default port 45661 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:36,499 WARN [IPC Server handler 2 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:36,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741856_1032 (size=45) 2024-11-21T08:13:36,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741856_1032 (size=45) 2024-11-21T08:13:36,505 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,sss,1732176816148.4ed3111e35326fc2c5455ae2475ab1b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:36,506 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 4ed3111e35326fc2c5455ae2475ab1b5, disabling compactions & flushes 2024-11-21T08:13:36,506 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,sss,1732176816148.4ed3111e35326fc2c5455ae2475ab1b5. 2024-11-21T08:13:36,506 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732176816148.4ed3111e35326fc2c5455ae2475ab1b5. 2024-11-21T08:13:36,506 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732176816148.4ed3111e35326fc2c5455ae2475ab1b5. after waiting 0 ms 2024-11-21T08:13:36,506 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732176816148.4ed3111e35326fc2c5455ae2475ab1b5. 2024-11-21T08:13:36,506 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,sss,1732176816148.4ed3111e35326fc2c5455ae2475ab1b5. 2024-11-21T08:13:36,506 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4ed3111e35326fc2c5455ae2475ab1b5: Waiting for close lock at 1732176816505Disabling compacts and flushes for region at 1732176816505Disabling writes for close at 1732176816506 (+1 ms)Writing region close event to WAL at 1732176816506Closed at 1732176816506 2024-11-21T08:13:36,506 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ec3a5c93577675c61d946ed41b85bdb5, NAME => 'test,ttt,1732176816148.ec3a5c93577675c61d946ed41b85bdb5.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T08:13:36,510 WARN [IPC Server handler 3 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:36,510 WARN [IPC Server handler 3 on default port 45661 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:36,510 WARN [IPC Server handler 3 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:36,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741857_1033 (size=45) 2024-11-21T08:13:36,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741857_1033 (size=45) 2024-11-21T08:13:36,517 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ttt,1732176816148.ec3a5c93577675c61d946ed41b85bdb5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:36,517 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing ec3a5c93577675c61d946ed41b85bdb5, disabling compactions & flushes 2024-11-21T08:13:36,517 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ttt,1732176816148.ec3a5c93577675c61d946ed41b85bdb5. 2024-11-21T08:13:36,517 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732176816148.ec3a5c93577675c61d946ed41b85bdb5. 2024-11-21T08:13:36,517 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732176816148.ec3a5c93577675c61d946ed41b85bdb5. after waiting 0 ms 2024-11-21T08:13:36,517 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732176816148.ec3a5c93577675c61d946ed41b85bdb5. 2024-11-21T08:13:36,517 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ttt,1732176816148.ec3a5c93577675c61d946ed41b85bdb5. 2024-11-21T08:13:36,517 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for ec3a5c93577675c61d946ed41b85bdb5: Waiting for close lock at 1732176816517Disabling compacts and flushes for region at 1732176816517Disabling writes for close at 1732176816517Writing region close event to WAL at 1732176816517Closed at 1732176816517 2024-11-21T08:13:36,518 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7ef5fca53c8ff0d12fc112ab68731575, NAME => 'test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:36,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741858_1034 (size=45) 2024-11-21T08:13:36,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741858_1034 (size=45) 2024-11-21T08:13:36,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741858_1034 (size=45) 2024-11-21T08:13:36,532 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:36,532 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 7ef5fca53c8ff0d12fc112ab68731575, disabling compactions & flushes 2024-11-21T08:13:36,532 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575. 2024-11-21T08:13:36,532 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575. 2024-11-21T08:13:36,532 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575. after waiting 0 ms 2024-11-21T08:13:36,532 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575. 2024-11-21T08:13:36,532 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575. 2024-11-21T08:13:36,532 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7ef5fca53c8ff0d12fc112ab68731575: Waiting for close lock at 1732176816532Disabling compacts and flushes for region at 1732176816532Disabling writes for close at 1732176816532Writing region close event to WAL at 1732176816532Closed at 1732176816532 2024-11-21T08:13:36,533 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8dfe569750c256f741c6d3e5ed072503, NAME => 'test,vvv,1732176816148.8dfe569750c256f741c6d3e5ed072503.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:36,538 WARN [IPC Server handler 3 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:36,538 WARN [IPC Server handler 3 on default port 45661 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:36,538 WARN [IPC Server handler 3 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:36,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741859_1035 (size=45) 2024-11-21T08:13:36,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741859_1035 (size=45) 2024-11-21T08:13:36,544 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,vvv,1732176816148.8dfe569750c256f741c6d3e5ed072503.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:36,544 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 8dfe569750c256f741c6d3e5ed072503, disabling compactions & flushes 2024-11-21T08:13:36,544 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,vvv,1732176816148.8dfe569750c256f741c6d3e5ed072503. 2024-11-21T08:13:36,544 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732176816148.8dfe569750c256f741c6d3e5ed072503. 2024-11-21T08:13:36,544 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732176816148.8dfe569750c256f741c6d3e5ed072503. after waiting 0 ms 2024-11-21T08:13:36,544 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732176816148.8dfe569750c256f741c6d3e5ed072503. 2024-11-21T08:13:36,544 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,vvv,1732176816148.8dfe569750c256f741c6d3e5ed072503. 2024-11-21T08:13:36,544 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8dfe569750c256f741c6d3e5ed072503: Waiting for close lock at 1732176816544Disabling compacts and flushes for region at 1732176816544Disabling writes for close at 1732176816544Writing region close event to WAL at 1732176816544Closed at 1732176816544 2024-11-21T08:13:36,545 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ebf8202a2e0e04a2c5c7f5f0e491d40e, NAME => 'test,www,1732176816148.ebf8202a2e0e04a2c5c7f5f0e491d40e.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:36,549 WARN [IPC Server handler 0 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:36,549 WARN [IPC Server handler 0 on default port 45661 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:36,549 WARN [IPC Server handler 0 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:36,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741860_1036 (size=45) 2024-11-21T08:13:36,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741860_1036 (size=45) 2024-11-21T08:13:36,556 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,www,1732176816148.ebf8202a2e0e04a2c5c7f5f0e491d40e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:36,556 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing ebf8202a2e0e04a2c5c7f5f0e491d40e, disabling compactions & flushes 2024-11-21T08:13:36,556 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,www,1732176816148.ebf8202a2e0e04a2c5c7f5f0e491d40e. 2024-11-21T08:13:36,556 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732176816148.ebf8202a2e0e04a2c5c7f5f0e491d40e. 2024-11-21T08:13:36,556 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1732176816148.ebf8202a2e0e04a2c5c7f5f0e491d40e. after waiting 0 ms 2024-11-21T08:13:36,556 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1732176816148.ebf8202a2e0e04a2c5c7f5f0e491d40e. 2024-11-21T08:13:36,556 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,www,1732176816148.ebf8202a2e0e04a2c5c7f5f0e491d40e. 2024-11-21T08:13:36,556 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for ebf8202a2e0e04a2c5c7f5f0e491d40e: Waiting for close lock at 1732176816556Disabling compacts and flushes for region at 1732176816556Disabling writes for close at 1732176816556Writing region close event to WAL at 1732176816556Closed at 1732176816556 2024-11-21T08:13:36,557 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 021eaa39c912c55ccf9b758a5c92d1f8, NAME => 'test,xxx,1732176816148.021eaa39c912c55ccf9b758a5c92d1f8.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:36,561 WARN [IPC Server handler 4 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:36,561 WARN [IPC Server handler 4 on default port 45661 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:36,561 WARN [IPC Server handler 4 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:36,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741861_1037 (size=45) 2024-11-21T08:13:36,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741861_1037 (size=45) 2024-11-21T08:13:36,568 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,xxx,1732176816148.021eaa39c912c55ccf9b758a5c92d1f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:36,568 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 021eaa39c912c55ccf9b758a5c92d1f8, disabling compactions & flushes 2024-11-21T08:13:36,568 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,xxx,1732176816148.021eaa39c912c55ccf9b758a5c92d1f8. 2024-11-21T08:13:36,568 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732176816148.021eaa39c912c55ccf9b758a5c92d1f8. 2024-11-21T08:13:36,568 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732176816148.021eaa39c912c55ccf9b758a5c92d1f8. after waiting 0 ms 2024-11-21T08:13:36,568 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732176816148.021eaa39c912c55ccf9b758a5c92d1f8. 2024-11-21T08:13:36,568 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,xxx,1732176816148.021eaa39c912c55ccf9b758a5c92d1f8. 2024-11-21T08:13:36,568 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 021eaa39c912c55ccf9b758a5c92d1f8: Waiting for close lock at 1732176816568Disabling compacts and flushes for region at 1732176816568Disabling writes for close at 1732176816568Writing region close event to WAL at 1732176816568Closed at 1732176816568 2024-11-21T08:13:36,569 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => af43494f9c57dd49b6d504756610f96c, NAME => 'test,yyy,1732176816148.af43494f9c57dd49b6d504756610f96c.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:36,572 WARN [IPC Server handler 0 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:36,572 WARN [IPC Server handler 0 on default port 45661 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:36,572 WARN [IPC Server handler 0 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:36,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741862_1038 (size=45) 2024-11-21T08:13:36,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741862_1038 (size=45) 2024-11-21T08:13:36,578 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,yyy,1732176816148.af43494f9c57dd49b6d504756610f96c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:36,578 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing af43494f9c57dd49b6d504756610f96c, disabling compactions & flushes 2024-11-21T08:13:36,578 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,yyy,1732176816148.af43494f9c57dd49b6d504756610f96c. 2024-11-21T08:13:36,578 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732176816148.af43494f9c57dd49b6d504756610f96c. 2024-11-21T08:13:36,579 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732176816148.af43494f9c57dd49b6d504756610f96c. after waiting 0 ms 2024-11-21T08:13:36,579 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732176816148.af43494f9c57dd49b6d504756610f96c. 2024-11-21T08:13:36,579 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,yyy,1732176816148.af43494f9c57dd49b6d504756610f96c. 2024-11-21T08:13:36,579 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for af43494f9c57dd49b6d504756610f96c: Waiting for close lock at 1732176816578Disabling compacts and flushes for region at 1732176816578Disabling writes for close at 1732176816579 (+1 ms)Writing region close event to WAL at 1732176816579Closed at 1732176816579 2024-11-21T08:13:36,579 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3589ece79012bcc3080c57b249902675, NAME => 'test,zzz,1732176816148.3589ece79012bcc3080c57b249902675.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db 2024-11-21T08:13:36,582 WARN [IPC Server handler 2 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-21T08:13:36,583 WARN [IPC Server handler 2 on default port 45661 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-21T08:13:36,583 WARN [IPC Server handler 2 on default port 45661 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-21T08:13:36,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741863_1039 (size=42) 2024-11-21T08:13:36,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741863_1039 (size=42) 2024-11-21T08:13:36,588 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,zzz,1732176816148.3589ece79012bcc3080c57b249902675.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:36,588 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 3589ece79012bcc3080c57b249902675, disabling compactions & flushes 2024-11-21T08:13:36,588 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,zzz,1732176816148.3589ece79012bcc3080c57b249902675. 2024-11-21T08:13:36,588 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732176816148.3589ece79012bcc3080c57b249902675. 2024-11-21T08:13:36,589 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732176816148.3589ece79012bcc3080c57b249902675. after waiting 0 ms 2024-11-21T08:13:36,589 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732176816148.3589ece79012bcc3080c57b249902675. 2024-11-21T08:13:36,589 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,zzz,1732176816148.3589ece79012bcc3080c57b249902675. 2024-11-21T08:13:36,589 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3589ece79012bcc3080c57b249902675: Waiting for close lock at 1732176816588Disabling compacts and flushes for region at 1732176816588Disabling writes for close at 1732176816589 (+1 ms)Writing region close event to WAL at 1732176816589Closed at 1732176816589 2024-11-21T08:13:36,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741827_1003 (size=196) 2024-11-21T08:13:36,676 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1732176816148.3773623806176f3d7f8efb3efe1b7937.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:36,676 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 3773623806176f3d7f8efb3efe1b7937, disabling compactions & flushes 2024-11-21T08:13:36,676 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1732176816148.3773623806176f3d7f8efb3efe1b7937. 2024-11-21T08:13:36,676 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732176816148.3773623806176f3d7f8efb3efe1b7937. 2024-11-21T08:13:36,676 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732176816148.3773623806176f3d7f8efb3efe1b7937. after waiting 0 ms 2024-11-21T08:13:36,676 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732176816148.3773623806176f3d7f8efb3efe1b7937. 2024-11-21T08:13:36,676 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1732176816148.3773623806176f3d7f8efb3efe1b7937. 2024-11-21T08:13:36,677 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 3773623806176f3d7f8efb3efe1b7937: Waiting for close lock at 1732176816676Disabling compacts and flushes for region at 1732176816676Disabling writes for close at 1732176816676Writing region close event to WAL at 1732176816676Closed at 1732176816676 2024-11-21T08:13:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T08:13:36,829 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,mmm,1732176816148.d72691fc99dffac743de3d6bcb0f84ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:36,829 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing d72691fc99dffac743de3d6bcb0f84ed, disabling compactions & flushes 2024-11-21T08:13:36,829 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,mmm,1732176816148.d72691fc99dffac743de3d6bcb0f84ed. 2024-11-21T08:13:36,829 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732176816148.d72691fc99dffac743de3d6bcb0f84ed. 2024-11-21T08:13:36,829 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732176816148.d72691fc99dffac743de3d6bcb0f84ed. after waiting 0 ms 2024-11-21T08:13:36,829 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732176816148.d72691fc99dffac743de3d6bcb0f84ed. 2024-11-21T08:13:36,829 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,mmm,1732176816148.d72691fc99dffac743de3d6bcb0f84ed. 2024-11-21T08:13:36,829 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for d72691fc99dffac743de3d6bcb0f84ed: Waiting for close lock at 1732176816829Disabling compacts and flushes for region at 1732176816829Disabling writes for close at 1732176816829Writing region close event to WAL at 1732176816829Closed at 1732176816829 2024-11-21T08:13:36,832 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T08:13:36,839 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1732176816148.fd0a67dc199543d2675c547a7cb41702.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732176816832"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732176816832"}]},"ts":"1732176816832"} 2024-11-21T08:13:36,839 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1732176816148.f38fd8b8defc002b7cb3de558717f27c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732176816832"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732176816832"}]},"ts":"1732176816832"} 2024-11-21T08:13:36,840 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1732176816148.4f43187c46588bff73d01ba7eb26a6f8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732176816832"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732176816832"}]},"ts":"1732176816832"} 2024-11-21T08:13:36,840 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1732176816148.cd29f056dad8c6c37d0979434891505d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732176816832"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732176816832"}]},"ts":"1732176816832"} 2024-11-21T08:13:36,840 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1732176816148.829529fdccf9853d96113fa67713ffbc.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732176816832"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732176816832"}]},"ts":"1732176816832"} 2024-11-21T08:13:36,840 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1732176816148.ee7ef14c3df8915e550cfda528563489.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732176816832"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732176816832"}]},"ts":"1732176816832"} 2024-11-21T08:13:36,841 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1732176816148.9eb0364d53b384c3e59ecc15348e6c3c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732176816832"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732176816832"}]},"ts":"1732176816832"} 2024-11-21T08:13:36,841 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1732176816148.fd97d8f813bdfcba3626cc4950c7c1e1.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732176816832"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732176816832"}]},"ts":"1732176816832"} 2024-11-21T08:13:36,841 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1732176816148.5fb158c8201cfbc5babdbfc33ee45ab1.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732176816832"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732176816832"}]},"ts":"1732176816832"} 2024-11-21T08:13:36,841 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1732176816148.dbfd5210cf46b2a246edeeebd244b662.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732176816832"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732176816832"}]},"ts":"1732176816832"} 2024-11-21T08:13:36,842 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1732176816148.94c604fd0d435e28ac643b49d6cbee14.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732176816832"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732176816832"}]},"ts":"1732176816832"} 2024-11-21T08:13:36,842 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1732176816148.99e7b3565f08f10e8c368072e9c21c09.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732176816832"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732176816832"}]},"ts":"1732176816832"} 2024-11-21T08:13:36,842 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1732176816148.477e65b3290a1e6e03e5eb59f7aa956d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732176816832"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732176816832"}]},"ts":"1732176816832"} 2024-11-21T08:13:36,842 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1732176816148.5017ae3acb74fe8c7d0b9c6d9a2cc402.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732176816832"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732176816832"}]},"ts":"1732176816832"} 2024-11-21T08:13:36,842 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1732176816148.2d5b1f002057972e76beae8f19094030.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732176816832"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732176816832"}]},"ts":"1732176816832"} 2024-11-21T08:13:36,843 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1732176816148.f55c025b8995c15117005602a775de64.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732176816832"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732176816832"}]},"ts":"1732176816832"} 2024-11-21T08:13:36,843 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1732176816148.4ed3111e35326fc2c5455ae2475ab1b5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732176816832"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732176816832"}]},"ts":"1732176816832"} 2024-11-21T08:13:36,843 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1732176816148.ec3a5c93577675c61d946ed41b85bdb5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732176816832"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732176816832"}]},"ts":"1732176816832"} 2024-11-21T08:13:36,843 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732176816832"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732176816832"}]},"ts":"1732176816832"} 2024-11-21T08:13:36,843 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1732176816148.8dfe569750c256f741c6d3e5ed072503.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732176816832"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732176816832"}]},"ts":"1732176816832"} 2024-11-21T08:13:36,843 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1732176816148.ebf8202a2e0e04a2c5c7f5f0e491d40e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732176816832"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732176816832"}]},"ts":"1732176816832"} 2024-11-21T08:13:36,843 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1732176816148.021eaa39c912c55ccf9b758a5c92d1f8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732176816832"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732176816832"}]},"ts":"1732176816832"} 2024-11-21T08:13:36,844 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1732176816148.af43494f9c57dd49b6d504756610f96c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732176816832"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732176816832"}]},"ts":"1732176816832"} 2024-11-21T08:13:36,844 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1732176816148.3589ece79012bcc3080c57b249902675.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732176816832"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732176816832"}]},"ts":"1732176816832"} 2024-11-21T08:13:36,844 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1732176816148.3773623806176f3d7f8efb3efe1b7937.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732176816832"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732176816832"}]},"ts":"1732176816832"} 2024-11-21T08:13:36,844 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1732176816148.d72691fc99dffac743de3d6bcb0f84ed.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732176816832"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732176816832"}]},"ts":"1732176816832"} 2024-11-21T08:13:36,889 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-21T08:13:36,893 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T08:13:36,895 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732176816893"}]},"ts":"1732176816893"} 2024-11-21T08:13:36,900 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-21T08:13:36,901 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {6ed56fb6cbf7=0} racks are {/default-rack=0} 2024-11-21T08:13:36,917 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T08:13:36,917 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T08:13:36,917 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-21T08:13:36,917 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T08:13:36,917 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T08:13:36,917 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-21T08:13:36,917 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T08:13:36,917 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T08:13:36,917 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-21T08:13:36,917 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T08:13:36,921 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f38fd8b8defc002b7cb3de558717f27c, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4f43187c46588bff73d01ba7eb26a6f8, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=829529fdccf9853d96113fa67713ffbc, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ee7ef14c3df8915e550cfda528563489, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fd97d8f813bdfcba3626cc4950c7c1e1, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5fb158c8201cfbc5babdbfc33ee45ab1, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=94c604fd0d435e28ac643b49d6cbee14, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d72691fc99dffac743de3d6bcb0f84ed, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=99e7b3565f08f10e8c368072e9c21c09, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5017ae3acb74fe8c7d0b9c6d9a2cc402, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2d5b1f002057972e76beae8f19094030, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4ed3111e35326fc2c5455ae2475ab1b5, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ec3a5c93577675c61d946ed41b85bdb5, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8dfe569750c256f741c6d3e5ed072503, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ebf8202a2e0e04a2c5c7f5f0e491d40e, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=af43494f9c57dd49b6d504756610f96c, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3589ece79012bcc3080c57b249902675, ASSIGN}] 2024-11-21T08:13:36,932 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3589ece79012bcc3080c57b249902675, ASSIGN 2024-11-21T08:13:36,934 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3589ece79012bcc3080c57b249902675, ASSIGN; state=OFFLINE, location=6ed56fb6cbf7,35295,1732176812677; forceNewPlan=false, retain=false 2024-11-21T08:13:36,935 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=af43494f9c57dd49b6d504756610f96c, ASSIGN 2024-11-21T08:13:36,944 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=af43494f9c57dd49b6d504756610f96c, ASSIGN; state=OFFLINE, location=6ed56fb6cbf7,38845,1732176812866; forceNewPlan=false, retain=false 2024-11-21T08:13:36,949 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:36,951 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN; state=OFFLINE, location=6ed56fb6cbf7,33091,1732176812968; forceNewPlan=false, retain=false 2024-11-21T08:13:36,953 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ebf8202a2e0e04a2c5c7f5f0e491d40e, ASSIGN 2024-11-21T08:13:36,954 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ebf8202a2e0e04a2c5c7f5f0e491d40e, ASSIGN; state=OFFLINE, location=6ed56fb6cbf7,35295,1732176812677; forceNewPlan=false, retain=false 2024-11-21T08:13:36,955 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8dfe569750c256f741c6d3e5ed072503, ASSIGN 2024-11-21T08:13:36,956 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8dfe569750c256f741c6d3e5ed072503, ASSIGN; state=OFFLINE, location=6ed56fb6cbf7,38845,1732176812866; forceNewPlan=false, retain=false 2024-11-21T08:13:36,958 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, ASSIGN 2024-11-21T08:13:36,959 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, ASSIGN; state=OFFLINE, location=6ed56fb6cbf7,33091,1732176812968; forceNewPlan=false, retain=false 2024-11-21T08:13:36,960 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ec3a5c93577675c61d946ed41b85bdb5, ASSIGN 2024-11-21T08:13:36,961 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ec3a5c93577675c61d946ed41b85bdb5, ASSIGN; state=OFFLINE, location=6ed56fb6cbf7,35295,1732176812677; forceNewPlan=false, retain=false 2024-11-21T08:13:36,963 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4ed3111e35326fc2c5455ae2475ab1b5, ASSIGN 2024-11-21T08:13:36,964 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4ed3111e35326fc2c5455ae2475ab1b5, ASSIGN; state=OFFLINE, location=6ed56fb6cbf7,38845,1732176812866; forceNewPlan=false, retain=false 2024-11-21T08:13:36,965 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN 2024-11-21T08:13:36,966 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN; state=OFFLINE, location=6ed56fb6cbf7,33091,1732176812968; forceNewPlan=false, retain=false 2024-11-21T08:13:36,968 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2d5b1f002057972e76beae8f19094030, ASSIGN 2024-11-21T08:13:36,970 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2d5b1f002057972e76beae8f19094030, ASSIGN; state=OFFLINE, location=6ed56fb6cbf7,35295,1732176812677; forceNewPlan=false, retain=false 2024-11-21T08:13:36,971 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5017ae3acb74fe8c7d0b9c6d9a2cc402, ASSIGN 2024-11-21T08:13:36,973 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5017ae3acb74fe8c7d0b9c6d9a2cc402, ASSIGN; state=OFFLINE, location=6ed56fb6cbf7,38845,1732176812866; forceNewPlan=false, retain=false 2024-11-21T08:13:36,975 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:36,976 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN; state=OFFLINE, location=6ed56fb6cbf7,33091,1732176812968; forceNewPlan=false, retain=false 2024-11-21T08:13:36,977 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=99e7b3565f08f10e8c368072e9c21c09, ASSIGN 2024-11-21T08:13:36,979 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=99e7b3565f08f10e8c368072e9c21c09, ASSIGN; state=OFFLINE, location=6ed56fb6cbf7,35295,1732176812677; forceNewPlan=false, retain=false 2024-11-21T08:13:36,980 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d72691fc99dffac743de3d6bcb0f84ed, ASSIGN 2024-11-21T08:13:36,982 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d72691fc99dffac743de3d6bcb0f84ed, ASSIGN; state=OFFLINE, location=6ed56fb6cbf7,38845,1732176812866; forceNewPlan=false, retain=false 2024-11-21T08:13:36,983 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=94c604fd0d435e28ac643b49d6cbee14, ASSIGN 2024-11-21T08:13:36,984 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=94c604fd0d435e28ac643b49d6cbee14, ASSIGN; state=OFFLINE, location=6ed56fb6cbf7,38845,1732176812866; forceNewPlan=false, retain=false 2024-11-21T08:13:36,986 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN 2024-11-21T08:13:36,988 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN; state=OFFLINE, location=6ed56fb6cbf7,33091,1732176812968; forceNewPlan=false, retain=false 2024-11-21T08:13:36,992 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5fb158c8201cfbc5babdbfc33ee45ab1, ASSIGN 2024-11-21T08:13:36,993 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5fb158c8201cfbc5babdbfc33ee45ab1, ASSIGN; state=OFFLINE, location=6ed56fb6cbf7,35295,1732176812677; forceNewPlan=false, retain=false 2024-11-21T08:13:36,995 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fd97d8f813bdfcba3626cc4950c7c1e1, ASSIGN 2024-11-21T08:13:36,997 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fd97d8f813bdfcba3626cc4950c7c1e1, ASSIGN; state=OFFLINE, location=6ed56fb6cbf7,38845,1732176812866; forceNewPlan=false, retain=false 2024-11-21T08:13:36,998 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN 2024-11-21T08:13:37,000 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN; state=OFFLINE, location=6ed56fb6cbf7,33091,1732176812968; forceNewPlan=false, retain=false 2024-11-21T08:13:37,001 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ee7ef14c3df8915e550cfda528563489, ASSIGN 2024-11-21T08:13:37,003 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ee7ef14c3df8915e550cfda528563489, ASSIGN; state=OFFLINE, location=6ed56fb6cbf7,35295,1732176812677; forceNewPlan=false, retain=false 2024-11-21T08:13:37,005 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=829529fdccf9853d96113fa67713ffbc, ASSIGN 2024-11-21T08:13:37,006 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=829529fdccf9853d96113fa67713ffbc, ASSIGN; state=OFFLINE, location=6ed56fb6cbf7,38845,1732176812866; forceNewPlan=false, retain=false 2024-11-21T08:13:37,008 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:37,010 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN; state=OFFLINE, location=6ed56fb6cbf7,33091,1732176812968; forceNewPlan=false, retain=false 2024-11-21T08:13:37,011 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4f43187c46588bff73d01ba7eb26a6f8, ASSIGN 2024-11-21T08:13:37,013 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4f43187c46588bff73d01ba7eb26a6f8, ASSIGN; state=OFFLINE, location=6ed56fb6cbf7,35295,1732176812677; forceNewPlan=false, retain=false 2024-11-21T08:13:37,015 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f38fd8b8defc002b7cb3de558717f27c, ASSIGN 2024-11-21T08:13:37,016 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f38fd8b8defc002b7cb3de558717f27c, ASSIGN; state=OFFLINE, location=6ed56fb6cbf7,38845,1732176812866; forceNewPlan=false, retain=false 2024-11-21T08:13:37,018 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN 2024-11-21T08:13:37,019 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN; state=OFFLINE, location=6ed56fb6cbf7,33091,1732176812968; forceNewPlan=false, retain=false 2024-11-21T08:13:37,021 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN 2024-11-21T08:13:37,022 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN; state=OFFLINE, location=6ed56fb6cbf7,33091,1732176812968; forceNewPlan=false, retain=false 2024-11-21T08:13:37,087 INFO [6ed56fb6cbf7:32901 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-21T08:13:37,089 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=af43494f9c57dd49b6d504756610f96c, regionState=OPENING, regionLocation=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:37,090 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=8dfe569750c256f741c6d3e5ed072503, regionState=OPENING, regionLocation=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:37,090 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=4ed3111e35326fc2c5455ae2475ab1b5, regionState=OPENING, regionLocation=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:37,091 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=5017ae3acb74fe8c7d0b9c6d9a2cc402, regionState=OPENING, regionLocation=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:37,092 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=d72691fc99dffac743de3d6bcb0f84ed, regionState=OPENING, regionLocation=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:37,092 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=94c604fd0d435e28ac643b49d6cbee14, regionState=OPENING, regionLocation=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:37,092 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=af43494f9c57dd49b6d504756610f96c, ASSIGN because future has completed 2024-11-21T08:13:37,093 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=fd97d8f813bdfcba3626cc4950c7c1e1, regionState=OPENING, regionLocation=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:37,093 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=829529fdccf9853d96113fa67713ffbc, regionState=OPENING, regionLocation=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:37,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure af43494f9c57dd49b6d504756610f96c, server=6ed56fb6cbf7,38845,1732176812866}] 2024-11-21T08:13:37,096 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8dfe569750c256f741c6d3e5ed072503, ASSIGN because future has completed 2024-11-21T08:13:37,098 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4ed3111e35326fc2c5455ae2475ab1b5, ASSIGN because future has completed 2024-11-21T08:13:37,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5017ae3acb74fe8c7d0b9c6d9a2cc402, ASSIGN because future has completed 2024-11-21T08:13:37,101 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d72691fc99dffac743de3d6bcb0f84ed, ASSIGN because future has completed 2024-11-21T08:13:37,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure d72691fc99dffac743de3d6bcb0f84ed, server=6ed56fb6cbf7,38845,1732176812866}] 2024-11-21T08:13:37,103 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=94c604fd0d435e28ac643b49d6cbee14, ASSIGN because future has completed 2024-11-21T08:13:37,105 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fd97d8f813bdfcba3626cc4950c7c1e1, ASSIGN because future has completed 2024-11-21T08:13:37,106 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=829529fdccf9853d96113fa67713ffbc, ASSIGN because future has completed 2024-11-21T08:13:37,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 829529fdccf9853d96113fa67713ffbc, server=6ed56fb6cbf7,38845,1732176812866}] 2024-11-21T08:13:37,115 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd97d8f813bdfcba3626cc4950c7c1e1, server=6ed56fb6cbf7,38845,1732176812866}] 2024-11-21T08:13:37,120 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94c604fd0d435e28ac643b49d6cbee14, server=6ed56fb6cbf7,38845,1732176812866}] 2024-11-21T08:13:37,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5017ae3acb74fe8c7d0b9c6d9a2cc402, server=6ed56fb6cbf7,38845,1732176812866}] 2024-11-21T08:13:37,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4ed3111e35326fc2c5455ae2475ab1b5, server=6ed56fb6cbf7,38845,1732176812866}] 2024-11-21T08:13:37,133 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8dfe569750c256f741c6d3e5ed072503, server=6ed56fb6cbf7,38845,1732176812866}] 2024-11-21T08:13:37,138 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f38fd8b8defc002b7cb3de558717f27c, regionState=OPENING, regionLocation=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:37,139 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=3589ece79012bcc3080c57b249902675, regionState=OPENING, regionLocation=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:37,139 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=ebf8202a2e0e04a2c5c7f5f0e491d40e, regionState=OPENING, regionLocation=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:37,140 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=ec3a5c93577675c61d946ed41b85bdb5, regionState=OPENING, regionLocation=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:37,141 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=2d5b1f002057972e76beae8f19094030, regionState=OPENING, regionLocation=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:37,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f38fd8b8defc002b7cb3de558717f27c, ASSIGN because future has completed 2024-11-21T08:13:37,142 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=99e7b3565f08f10e8c368072e9c21c09, regionState=OPENING, regionLocation=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:37,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure f38fd8b8defc002b7cb3de558717f27c, server=6ed56fb6cbf7,38845,1732176812866}] 2024-11-21T08:13:37,143 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3589ece79012bcc3080c57b249902675, ASSIGN because future has completed 2024-11-21T08:13:37,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ebf8202a2e0e04a2c5c7f5f0e491d40e, ASSIGN because future has completed 2024-11-21T08:13:37,147 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ec3a5c93577675c61d946ed41b85bdb5, ASSIGN because future has completed 2024-11-21T08:13:37,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure ec3a5c93577675c61d946ed41b85bdb5, server=6ed56fb6cbf7,35295,1732176812677}] 2024-11-21T08:13:37,148 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2d5b1f002057972e76beae8f19094030, ASSIGN because future has completed 2024-11-21T08:13:37,150 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=99e7b3565f08f10e8c368072e9c21c09, ASSIGN because future has completed 2024-11-21T08:13:37,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 99e7b3565f08f10e8c368072e9c21c09, server=6ed56fb6cbf7,35295,1732176812677}] 2024-11-21T08:13:37,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d5b1f002057972e76beae8f19094030, server=6ed56fb6cbf7,35295,1732176812677}] 2024-11-21T08:13:37,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure ebf8202a2e0e04a2c5c7f5f0e491d40e, server=6ed56fb6cbf7,35295,1732176812677}] 2024-11-21T08:13:37,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3589ece79012bcc3080c57b249902675, server=6ed56fb6cbf7,35295,1732176812677}] 2024-11-21T08:13:37,173 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=5fb158c8201cfbc5babdbfc33ee45ab1, regionState=OPENING, regionLocation=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:37,173 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=ee7ef14c3df8915e550cfda528563489, regionState=OPENING, regionLocation=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:37,174 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=4f43187c46588bff73d01ba7eb26a6f8, regionState=OPENING, regionLocation=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:37,175 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=021eaa39c912c55ccf9b758a5c92d1f8, regionState=OPENING, regionLocation=6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:37,176 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=7ef5fca53c8ff0d12fc112ab68731575, regionState=OPENING, regionLocation=6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:37,176 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5fb158c8201cfbc5babdbfc33ee45ab1, ASSIGN because future has completed 2024-11-21T08:13:37,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5fb158c8201cfbc5babdbfc33ee45ab1, server=6ed56fb6cbf7,35295,1732176812677}] 2024-11-21T08:13:37,178 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ee7ef14c3df8915e550cfda528563489, ASSIGN because future has completed 2024-11-21T08:13:37,179 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4f43187c46588bff73d01ba7eb26a6f8, ASSIGN because future has completed 2024-11-21T08:13:37,181 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN because future has completed 2024-11-21T08:13:37,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, ASSIGN because future has completed 2024-11-21T08:13:37,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ef5fca53c8ff0d12fc112ab68731575, server=6ed56fb6cbf7,33091,1732176812968}] 2024-11-21T08:13:37,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 021eaa39c912c55ccf9b758a5c92d1f8, server=6ed56fb6cbf7,33091,1732176812968}] 2024-11-21T08:13:37,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f43187c46588bff73d01ba7eb26a6f8, server=6ed56fb6cbf7,35295,1732176812677}] 2024-11-21T08:13:37,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee7ef14c3df8915e550cfda528563489, server=6ed56fb6cbf7,35295,1732176812677}] 2024-11-21T08:13:37,217 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=f55c025b8995c15117005602a775de64, regionState=OPENING, regionLocation=6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:37,218 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=477e65b3290a1e6e03e5eb59f7aa956d, regionState=OPENING, regionLocation=6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:37,219 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=dbfd5210cf46b2a246edeeebd244b662, regionState=OPENING, regionLocation=6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:37,220 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=9eb0364d53b384c3e59ecc15348e6c3c, regionState=OPENING, regionLocation=6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:37,220 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN because future has completed 2024-11-21T08:13:37,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure f55c025b8995c15117005602a775de64, server=6ed56fb6cbf7,33091,1732176812968}] 2024-11-21T08:13:37,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN because future has completed 2024-11-21T08:13:37,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN because future has completed 2024-11-21T08:13:37,225 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN because future has completed 2024-11-21T08:13:37,227 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9eb0364d53b384c3e59ecc15348e6c3c, server=6ed56fb6cbf7,33091,1732176812968}] 2024-11-21T08:13:37,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure dbfd5210cf46b2a246edeeebd244b662, server=6ed56fb6cbf7,33091,1732176812968}] 2024-11-21T08:13:37,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 477e65b3290a1e6e03e5eb59f7aa956d, server=6ed56fb6cbf7,33091,1732176812968}] 2024-11-21T08:13:37,239 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=cd29f056dad8c6c37d0979434891505d, regionState=OPENING, regionLocation=6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:37,240 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=3773623806176f3d7f8efb3efe1b7937, regionState=OPENING, regionLocation=6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:37,240 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fd0a67dc199543d2675c547a7cb41702, regionState=OPENING, regionLocation=6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:37,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN because future has completed 2024-11-21T08:13:37,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,33091,1732176812968}] 2024-11-21T08:13:37,245 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN because future has completed 2024-11-21T08:13:37,247 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN because future has completed 2024-11-21T08:13:37,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd0a67dc199543d2675c547a7cb41702, server=6ed56fb6cbf7,33091,1732176812968}] 2024-11-21T08:13:37,250 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T08:13:37,252 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35311, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T08:13:37,261 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,iii,1732176816148.fd97d8f813bdfcba3626cc4950c7c1e1. 2024-11-21T08:13:37,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3773623806176f3d7f8efb3efe1b7937, server=6ed56fb6cbf7,33091,1732176812968}] 2024-11-21T08:13:37,261 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => fd97d8f813bdfcba3626cc4950c7c1e1, NAME => 'test,iii,1732176816148.fd97d8f813bdfcba3626cc4950c7c1e1.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-21T08:13:37,262 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fd97d8f813bdfcba3626cc4950c7c1e1 2024-11-21T08:13:37,262 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,iii,1732176816148.fd97d8f813bdfcba3626cc4950c7c1e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:37,262 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for fd97d8f813bdfcba3626cc4950c7c1e1 2024-11-21T08:13:37,262 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for fd97d8f813bdfcba3626cc4950c7c1e1 2024-11-21T08:13:37,265 INFO [StoreOpener-fd97d8f813bdfcba3626cc4950c7c1e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fd97d8f813bdfcba3626cc4950c7c1e1 2024-11-21T08:13:37,268 INFO [StoreOpener-fd97d8f813bdfcba3626cc4950c7c1e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd97d8f813bdfcba3626cc4950c7c1e1 columnFamilyName family 2024-11-21T08:13:37,269 DEBUG [StoreOpener-fd97d8f813bdfcba3626cc4950c7c1e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:37,270 INFO [StoreOpener-fd97d8f813bdfcba3626cc4950c7c1e1-1 {}] regionserver.HStore(327): Store=fd97d8f813bdfcba3626cc4950c7c1e1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:37,270 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for fd97d8f813bdfcba3626cc4950c7c1e1 2024-11-21T08:13:37,272 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/fd97d8f813bdfcba3626cc4950c7c1e1 2024-11-21T08:13:37,273 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/fd97d8f813bdfcba3626cc4950c7c1e1 2024-11-21T08:13:37,274 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for fd97d8f813bdfcba3626cc4950c7c1e1 2024-11-21T08:13:37,274 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for fd97d8f813bdfcba3626cc4950c7c1e1 2024-11-21T08:13:37,278 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for fd97d8f813bdfcba3626cc4950c7c1e1 2024-11-21T08:13:37,281 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/fd97d8f813bdfcba3626cc4950c7c1e1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T08:13:37,283 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened fd97d8f813bdfcba3626cc4950c7c1e1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68327385, jitterRate=0.01815737783908844}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:37,283 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fd97d8f813bdfcba3626cc4950c7c1e1 2024-11-21T08:13:37,284 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for fd97d8f813bdfcba3626cc4950c7c1e1: Running coprocessor pre-open hook at 1732176817263Writing region info on filesystem at 1732176817263Initializing all the Stores at 1732176817265 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176817265Cleaning up temporary data from old regions at 1732176817274 (+9 ms)Running coprocessor post-open hooks at 1732176817283 (+9 ms)Region opened successfully at 1732176817284 (+1 ms) 2024-11-21T08:13:37,286 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1732176816148.fd97d8f813bdfcba3626cc4950c7c1e1., pid=34, masterSystemTime=1732176817249 2024-11-21T08:13:37,289 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1732176816148.fd97d8f813bdfcba3626cc4950c7c1e1. 2024-11-21T08:13:37,290 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,iii,1732176816148.fd97d8f813bdfcba3626cc4950c7c1e1. 2024-11-21T08:13:37,290 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,vvv,1732176816148.8dfe569750c256f741c6d3e5ed072503. 2024-11-21T08:13:37,290 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 8dfe569750c256f741c6d3e5ed072503, NAME => 'test,vvv,1732176816148.8dfe569750c256f741c6d3e5ed072503.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-21T08:13:37,290 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=fd97d8f813bdfcba3626cc4950c7c1e1, regionState=OPEN, openSeqNum=2, regionLocation=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:37,291 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8dfe569750c256f741c6d3e5ed072503 2024-11-21T08:13:37,291 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,vvv,1732176816148.8dfe569750c256f741c6d3e5ed072503.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:37,291 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 8dfe569750c256f741c6d3e5ed072503 2024-11-21T08:13:37,291 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 8dfe569750c256f741c6d3e5ed072503 2024-11-21T08:13:37,294 INFO [StoreOpener-8dfe569750c256f741c6d3e5ed072503-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8dfe569750c256f741c6d3e5ed072503 2024-11-21T08:13:37,296 INFO [StoreOpener-8dfe569750c256f741c6d3e5ed072503-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8dfe569750c256f741c6d3e5ed072503 columnFamilyName family 2024-11-21T08:13:37,296 DEBUG [StoreOpener-8dfe569750c256f741c6d3e5ed072503-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:37,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd97d8f813bdfcba3626cc4950c7c1e1, server=6ed56fb6cbf7,38845,1732176812866 because future has completed 2024-11-21T08:13:37,297 INFO [StoreOpener-8dfe569750c256f741c6d3e5ed072503-1 {}] regionserver.HStore(327): Store=8dfe569750c256f741c6d3e5ed072503/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:37,298 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 8dfe569750c256f741c6d3e5ed072503 2024-11-21T08:13:37,299 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/8dfe569750c256f741c6d3e5ed072503 2024-11-21T08:13:37,300 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/8dfe569750c256f741c6d3e5ed072503 2024-11-21T08:13:37,301 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 8dfe569750c256f741c6d3e5ed072503 2024-11-21T08:13:37,301 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 8dfe569750c256f741c6d3e5ed072503 2024-11-21T08:13:37,303 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T08:13:37,304 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 8dfe569750c256f741c6d3e5ed072503 2024-11-21T08:13:37,305 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39837, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T08:13:37,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=13 2024-11-21T08:13:37,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure fd97d8f813bdfcba3626cc4950c7c1e1, server=6ed56fb6cbf7,38845,1732176812866 in 183 msec 2024-11-21T08:13:37,309 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/8dfe569750c256f741c6d3e5ed072503/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T08:13:37,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fd97d8f813bdfcba3626cc4950c7c1e1, ASSIGN in 387 msec 2024-11-21T08:13:37,310 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 8dfe569750c256f741c6d3e5ed072503; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62439226, jitterRate=-0.06958302855491638}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:37,310 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8dfe569750c256f741c6d3e5ed072503 2024-11-21T08:13:37,310 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 8dfe569750c256f741c6d3e5ed072503: Running coprocessor pre-open hook at 1732176817291Writing region info on filesystem at 1732176817291Initializing all the Stores at 1732176817293 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176817293Cleaning up temporary data from old regions at 1732176817301 (+8 ms)Running coprocessor post-open hooks at 1732176817310 (+9 ms)Region opened successfully at 1732176817310 2024-11-21T08:13:37,311 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732176816148.8dfe569750c256f741c6d3e5ed072503., pid=38, masterSystemTime=1732176817249 2024-11-21T08:13:37,312 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,zzz,1732176816148.3589ece79012bcc3080c57b249902675. 2024-11-21T08:13:37,312 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 3589ece79012bcc3080c57b249902675, NAME => 'test,zzz,1732176816148.3589ece79012bcc3080c57b249902675.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-21T08:13:37,312 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3589ece79012bcc3080c57b249902675 2024-11-21T08:13:37,313 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,zzz,1732176816148.3589ece79012bcc3080c57b249902675.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:37,313 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 3589ece79012bcc3080c57b249902675 2024-11-21T08:13:37,313 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 3589ece79012bcc3080c57b249902675 2024-11-21T08:13:37,315 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1732176816148.8dfe569750c256f741c6d3e5ed072503. 2024-11-21T08:13:37,315 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,vvv,1732176816148.8dfe569750c256f741c6d3e5ed072503. 2024-11-21T08:13:37,315 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,lll,1732176816148.94c604fd0d435e28ac643b49d6cbee14. 2024-11-21T08:13:37,315 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 94c604fd0d435e28ac643b49d6cbee14, NAME => 'test,lll,1732176816148.94c604fd0d435e28ac643b49d6cbee14.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-21T08:13:37,315 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 94c604fd0d435e28ac643b49d6cbee14 2024-11-21T08:13:37,315 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,lll,1732176816148.94c604fd0d435e28ac643b49d6cbee14.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:37,316 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 94c604fd0d435e28ac643b49d6cbee14 2024-11-21T08:13:37,316 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=8dfe569750c256f741c6d3e5ed072503, regionState=OPEN, openSeqNum=2, regionLocation=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:37,316 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 94c604fd0d435e28ac643b49d6cbee14 2024-11-21T08:13:37,316 INFO [StoreOpener-3589ece79012bcc3080c57b249902675-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3589ece79012bcc3080c57b249902675 2024-11-21T08:13:37,319 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8dfe569750c256f741c6d3e5ed072503, server=6ed56fb6cbf7,38845,1732176812866 because future has completed 2024-11-21T08:13:37,320 INFO [StoreOpener-94c604fd0d435e28ac643b49d6cbee14-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 94c604fd0d435e28ac643b49d6cbee14 2024-11-21T08:13:37,323 INFO [StoreOpener-94c604fd0d435e28ac643b49d6cbee14-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94c604fd0d435e28ac643b49d6cbee14 columnFamilyName family 2024-11-21T08:13:37,323 DEBUG [StoreOpener-94c604fd0d435e28ac643b49d6cbee14-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:37,324 INFO [StoreOpener-94c604fd0d435e28ac643b49d6cbee14-1 {}] regionserver.HStore(327): Store=94c604fd0d435e28ac643b49d6cbee14/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:37,324 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 94c604fd0d435e28ac643b49d6cbee14 2024-11-21T08:13:37,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=26 2024-11-21T08:13:37,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 8dfe569750c256f741c6d3e5ed072503, server=6ed56fb6cbf7,38845,1732176812866 in 187 msec 2024-11-21T08:13:37,325 INFO [StoreOpener-3589ece79012bcc3080c57b249902675-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3589ece79012bcc3080c57b249902675 columnFamilyName family 2024-11-21T08:13:37,326 DEBUG [StoreOpener-3589ece79012bcc3080c57b249902675-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:37,326 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/94c604fd0d435e28ac643b49d6cbee14 2024-11-21T08:13:37,327 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/94c604fd0d435e28ac643b49d6cbee14 2024-11-21T08:13:37,327 INFO [StoreOpener-3589ece79012bcc3080c57b249902675-1 {}] regionserver.HStore(327): Store=3589ece79012bcc3080c57b249902675/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:37,327 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 3589ece79012bcc3080c57b249902675 2024-11-21T08:13:37,328 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 94c604fd0d435e28ac643b49d6cbee14 2024-11-21T08:13:37,328 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 94c604fd0d435e28ac643b49d6cbee14 2024-11-21T08:13:37,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8dfe569750c256f741c6d3e5ed072503, ASSIGN in 405 msec 2024-11-21T08:13:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T08:13:37,329 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/3589ece79012bcc3080c57b249902675 2024-11-21T08:13:37,330 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/3589ece79012bcc3080c57b249902675 2024-11-21T08:13:37,330 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 3589ece79012bcc3080c57b249902675 2024-11-21T08:13:37,330 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 3589ece79012bcc3080c57b249902675 2024-11-21T08:13:37,331 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 94c604fd0d435e28ac643b49d6cbee14 2024-11-21T08:13:37,333 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 3589ece79012bcc3080c57b249902675 2024-11-21T08:13:37,337 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/94c604fd0d435e28ac643b49d6cbee14/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T08:13:37,337 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/3589ece79012bcc3080c57b249902675/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T08:13:37,338 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 3589ece79012bcc3080c57b249902675; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66028860, jitterRate=-0.016093313694000244}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:37,338 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3589ece79012bcc3080c57b249902675 2024-11-21T08:13:37,338 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 94c604fd0d435e28ac643b49d6cbee14; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67006316, jitterRate=-0.0015280842781066895}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:37,338 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 94c604fd0d435e28ac643b49d6cbee14 2024-11-21T08:13:37,338 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 3589ece79012bcc3080c57b249902675: Running coprocessor pre-open hook at 1732176817313Writing region info on filesystem at 1732176817313Initializing all the Stores at 1732176817314 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176817314Cleaning up temporary data from old regions at 1732176817330 (+16 ms)Running coprocessor post-open hooks at 1732176817338 (+8 ms)Region opened successfully at 1732176817338 2024-11-21T08:13:37,338 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 94c604fd0d435e28ac643b49d6cbee14: Running coprocessor pre-open hook at 1732176817316Writing region info on filesystem at 1732176817316Initializing all the Stores at 1732176817317 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176817317Cleaning up temporary data from old regions at 1732176817328 (+11 ms)Running coprocessor post-open hooks at 1732176817338 (+10 ms)Region opened successfully at 1732176817338 2024-11-21T08:13:37,339 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732176816148.94c604fd0d435e28ac643b49d6cbee14., pid=35, masterSystemTime=1732176817249 2024-11-21T08:13:37,339 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1732176816148.3589ece79012bcc3080c57b249902675., pid=44, masterSystemTime=1732176817302 2024-11-21T08:13:37,343 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1732176816148.3589ece79012bcc3080c57b249902675. 2024-11-21T08:13:37,343 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,zzz,1732176816148.3589ece79012bcc3080c57b249902675. 2024-11-21T08:13:37,344 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,nnn,1732176816148.99e7b3565f08f10e8c368072e9c21c09. 2024-11-21T08:13:37,344 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 99e7b3565f08f10e8c368072e9c21c09, NAME => 'test,nnn,1732176816148.99e7b3565f08f10e8c368072e9c21c09.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-21T08:13:37,344 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=3589ece79012bcc3080c57b249902675, regionState=OPEN, openSeqNum=2, regionLocation=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:37,345 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 99e7b3565f08f10e8c368072e9c21c09 2024-11-21T08:13:37,345 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,nnn,1732176816148.99e7b3565f08f10e8c368072e9c21c09.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:37,345 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 99e7b3565f08f10e8c368072e9c21c09 2024-11-21T08:13:37,345 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 99e7b3565f08f10e8c368072e9c21c09 2024-11-21T08:13:37,346 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732176816148.94c604fd0d435e28ac643b49d6cbee14. 2024-11-21T08:13:37,346 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,lll,1732176816148.94c604fd0d435e28ac643b49d6cbee14. 2024-11-21T08:13:37,347 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,mmm,1732176816148.d72691fc99dffac743de3d6bcb0f84ed. 2024-11-21T08:13:37,347 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => d72691fc99dffac743de3d6bcb0f84ed, NAME => 'test,mmm,1732176816148.d72691fc99dffac743de3d6bcb0f84ed.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-21T08:13:37,347 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d72691fc99dffac743de3d6bcb0f84ed 2024-11-21T08:13:37,348 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,mmm,1732176816148.d72691fc99dffac743de3d6bcb0f84ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:37,348 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for d72691fc99dffac743de3d6bcb0f84ed 2024-11-21T08:13:37,348 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for d72691fc99dffac743de3d6bcb0f84ed 2024-11-21T08:13:37,348 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3589ece79012bcc3080c57b249902675, server=6ed56fb6cbf7,35295,1732176812677 because future has completed 2024-11-21T08:13:37,349 INFO [StoreOpener-99e7b3565f08f10e8c368072e9c21c09-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 99e7b3565f08f10e8c368072e9c21c09 2024-11-21T08:13:37,349 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=94c604fd0d435e28ac643b49d6cbee14, regionState=OPEN, openSeqNum=2, regionLocation=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:37,351 INFO [StoreOpener-99e7b3565f08f10e8c368072e9c21c09-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99e7b3565f08f10e8c368072e9c21c09 columnFamilyName family 2024-11-21T08:13:37,351 DEBUG [StoreOpener-99e7b3565f08f10e8c368072e9c21c09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:37,352 INFO [StoreOpener-d72691fc99dffac743de3d6bcb0f84ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d72691fc99dffac743de3d6bcb0f84ed 2024-11-21T08:13:37,353 INFO [StoreOpener-99e7b3565f08f10e8c368072e9c21c09-1 {}] regionserver.HStore(327): Store=99e7b3565f08f10e8c368072e9c21c09/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:37,353 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 99e7b3565f08f10e8c368072e9c21c09 2024-11-21T08:13:37,353 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94c604fd0d435e28ac643b49d6cbee14, server=6ed56fb6cbf7,38845,1732176812866 because future has completed 2024-11-21T08:13:37,354 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/99e7b3565f08f10e8c368072e9c21c09 2024-11-21T08:13:37,355 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575. 2024-11-21T08:13:37,355 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 7ef5fca53c8ff0d12fc112ab68731575, NAME => 'test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-21T08:13:37,355 INFO [StoreOpener-d72691fc99dffac743de3d6bcb0f84ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d72691fc99dffac743de3d6bcb0f84ed columnFamilyName family 2024-11-21T08:13:37,355 DEBUG [StoreOpener-d72691fc99dffac743de3d6bcb0f84ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:37,355 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7ef5fca53c8ff0d12fc112ab68731575 2024-11-21T08:13:37,356 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:37,356 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/99e7b3565f08f10e8c368072e9c21c09 2024-11-21T08:13:37,356 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 7ef5fca53c8ff0d12fc112ab68731575 2024-11-21T08:13:37,356 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 7ef5fca53c8ff0d12fc112ab68731575 2024-11-21T08:13:37,356 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 99e7b3565f08f10e8c368072e9c21c09 2024-11-21T08:13:37,357 INFO [StoreOpener-d72691fc99dffac743de3d6bcb0f84ed-1 {}] regionserver.HStore(327): Store=d72691fc99dffac743de3d6bcb0f84ed/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:37,357 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 99e7b3565f08f10e8c368072e9c21c09 2024-11-21T08:13:37,357 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for d72691fc99dffac743de3d6bcb0f84ed 2024-11-21T08:13:37,358 INFO [StoreOpener-7ef5fca53c8ff0d12fc112ab68731575-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7ef5fca53c8ff0d12fc112ab68731575 2024-11-21T08:13:37,358 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/d72691fc99dffac743de3d6bcb0f84ed 2024-11-21T08:13:37,359 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=30 2024-11-21T08:13:37,359 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 3589ece79012bcc3080c57b249902675, server=6ed56fb6cbf7,35295,1732176812677 in 188 msec 2024-11-21T08:13:37,359 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/d72691fc99dffac743de3d6bcb0f84ed 2024-11-21T08:13:37,359 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 99e7b3565f08f10e8c368072e9c21c09 2024-11-21T08:13:37,360 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for d72691fc99dffac743de3d6bcb0f84ed 2024-11-21T08:13:37,360 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for d72691fc99dffac743de3d6bcb0f84ed 2024-11-21T08:13:37,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3589ece79012bcc3080c57b249902675, ASSIGN in 439 msec 2024-11-21T08:13:37,363 INFO [StoreOpener-7ef5fca53c8ff0d12fc112ab68731575-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7ef5fca53c8ff0d12fc112ab68731575 columnFamilyName family 2024-11-21T08:13:37,363 DEBUG [StoreOpener-7ef5fca53c8ff0d12fc112ab68731575-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:37,363 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for d72691fc99dffac743de3d6bcb0f84ed 2024-11-21T08:13:37,364 INFO [StoreOpener-7ef5fca53c8ff0d12fc112ab68731575-1 {}] regionserver.HStore(327): Store=7ef5fca53c8ff0d12fc112ab68731575/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:37,364 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 7ef5fca53c8ff0d12fc112ab68731575 2024-11-21T08:13:37,366 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/7ef5fca53c8ff0d12fc112ab68731575 2024-11-21T08:13:37,366 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/7ef5fca53c8ff0d12fc112ab68731575 2024-11-21T08:13:37,367 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/d72691fc99dffac743de3d6bcb0f84ed/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T08:13:37,367 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 7ef5fca53c8ff0d12fc112ab68731575 2024-11-21T08:13:37,367 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 7ef5fca53c8ff0d12fc112ab68731575 2024-11-21T08:13:37,367 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened d72691fc99dffac743de3d6bcb0f84ed; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74014106, jitterRate=0.10289612412452698}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:37,367 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d72691fc99dffac743de3d6bcb0f84ed 2024-11-21T08:13:37,368 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for d72691fc99dffac743de3d6bcb0f84ed: Running coprocessor pre-open hook at 1732176817348Writing region info on filesystem at 1732176817348Initializing all the Stores at 1732176817351 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176817351Cleaning up temporary data from old regions at 1732176817360 (+9 ms)Running coprocessor post-open hooks at 1732176817368 (+8 ms)Region opened successfully at 1732176817368 2024-11-21T08:13:37,368 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/99e7b3565f08f10e8c368072e9c21c09/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T08:13:37,369 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 99e7b3565f08f10e8c368072e9c21c09; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60125178, jitterRate=-0.10406503081321716}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:37,369 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1732176816148.d72691fc99dffac743de3d6bcb0f84ed., pid=32, masterSystemTime=1732176817249 2024-11-21T08:13:37,369 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 99e7b3565f08f10e8c368072e9c21c09 2024-11-21T08:13:37,369 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 99e7b3565f08f10e8c368072e9c21c09: Running coprocessor pre-open hook at 1732176817345Writing region info on filesystem at 1732176817345Initializing all the Stores at 1732176817348 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176817348Cleaning up temporary data from old regions at 1732176817357 (+9 ms)Running coprocessor post-open hooks at 1732176817369 (+12 ms)Region opened successfully at 1732176817369 2024-11-21T08:13:37,371 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1732176816148.99e7b3565f08f10e8c368072e9c21c09., pid=41, masterSystemTime=1732176817302 2024-11-21T08:13:37,371 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 7ef5fca53c8ff0d12fc112ab68731575 2024-11-21T08:13:37,372 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1732176816148.d72691fc99dffac743de3d6bcb0f84ed. 2024-11-21T08:13:37,373 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,mmm,1732176816148.d72691fc99dffac743de3d6bcb0f84ed. 2024-11-21T08:13:37,373 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,fff,1732176816148.829529fdccf9853d96113fa67713ffbc. 2024-11-21T08:13:37,373 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 829529fdccf9853d96113fa67713ffbc, NAME => 'test,fff,1732176816148.829529fdccf9853d96113fa67713ffbc.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-21T08:13:37,374 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 829529fdccf9853d96113fa67713ffbc 2024-11-21T08:13:37,374 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,fff,1732176816148.829529fdccf9853d96113fa67713ffbc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:37,375 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 829529fdccf9853d96113fa67713ffbc 2024-11-21T08:13:37,375 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 829529fdccf9853d96113fa67713ffbc 2024-11-21T08:13:37,376 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1732176816148.99e7b3565f08f10e8c368072e9c21c09. 2024-11-21T08:13:37,377 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,nnn,1732176816148.99e7b3565f08f10e8c368072e9c21c09. 2024-11-21T08:13:37,377 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,www,1732176816148.ebf8202a2e0e04a2c5c7f5f0e491d40e. 2024-11-21T08:13:37,377 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/7ef5fca53c8ff0d12fc112ab68731575/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T08:13:37,377 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => ebf8202a2e0e04a2c5c7f5f0e491d40e, NAME => 'test,www,1732176816148.ebf8202a2e0e04a2c5c7f5f0e491d40e.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-21T08:13:37,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=16 2024-11-21T08:13:37,378 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ebf8202a2e0e04a2c5c7f5f0e491d40e 2024-11-21T08:13:37,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 94c604fd0d435e28ac643b49d6cbee14, server=6ed56fb6cbf7,38845,1732176812866 in 249 msec 2024-11-21T08:13:37,378 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,www,1732176816148.ebf8202a2e0e04a2c5c7f5f0e491d40e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:37,378 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for ebf8202a2e0e04a2c5c7f5f0e491d40e 2024-11-21T08:13:37,378 INFO [StoreOpener-829529fdccf9853d96113fa67713ffbc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 829529fdccf9853d96113fa67713ffbc 2024-11-21T08:13:37,378 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for ebf8202a2e0e04a2c5c7f5f0e491d40e 2024-11-21T08:13:37,378 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 7ef5fca53c8ff0d12fc112ab68731575; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74731336, jitterRate=0.11358368396759033}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:37,378 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7ef5fca53c8ff0d12fc112ab68731575 2024-11-21T08:13:37,379 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 7ef5fca53c8ff0d12fc112ab68731575: Running coprocessor pre-open hook at 1732176817356Writing region info on filesystem at 1732176817356Initializing all the Stores at 1732176817358 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176817358Cleaning up temporary data from old regions at 1732176817367 (+9 ms)Running coprocessor post-open hooks at 1732176817378 (+11 ms)Region opened successfully at 1732176817378 2024-11-21T08:13:37,380 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575., pid=46, masterSystemTime=1732176817346 2024-11-21T08:13:37,381 INFO [StoreOpener-ebf8202a2e0e04a2c5c7f5f0e491d40e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ebf8202a2e0e04a2c5c7f5f0e491d40e 2024-11-21T08:13:37,381 INFO [StoreOpener-829529fdccf9853d96113fa67713ffbc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 829529fdccf9853d96113fa67713ffbc columnFamilyName family 2024-11-21T08:13:37,381 DEBUG [StoreOpener-829529fdccf9853d96113fa67713ffbc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:37,382 INFO [StoreOpener-829529fdccf9853d96113fa67713ffbc-1 {}] regionserver.HStore(327): Store=829529fdccf9853d96113fa67713ffbc/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:37,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=94c604fd0d435e28ac643b49d6cbee14, ASSIGN in 458 msec 2024-11-21T08:13:37,383 INFO [StoreOpener-ebf8202a2e0e04a2c5c7f5f0e491d40e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ebf8202a2e0e04a2c5c7f5f0e491d40e columnFamilyName family 2024-11-21T08:13:37,383 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575. 2024-11-21T08:13:37,383 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575. 2024-11-21T08:13:37,383 DEBUG [StoreOpener-ebf8202a2e0e04a2c5c7f5f0e491d40e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:37,383 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 829529fdccf9853d96113fa67713ffbc 2024-11-21T08:13:37,384 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,eee,1732176816148.cd29f056dad8c6c37d0979434891505d. 2024-11-21T08:13:37,384 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => cd29f056dad8c6c37d0979434891505d, NAME => 'test,eee,1732176816148.cd29f056dad8c6c37d0979434891505d.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-21T08:13:37,385 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test cd29f056dad8c6c37d0979434891505d 2024-11-21T08:13:37,385 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,eee,1732176816148.cd29f056dad8c6c37d0979434891505d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:37,385 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for cd29f056dad8c6c37d0979434891505d 2024-11-21T08:13:37,385 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=7ef5fca53c8ff0d12fc112ab68731575, regionState=OPEN, openSeqNum=2, regionLocation=6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:37,385 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for cd29f056dad8c6c37d0979434891505d 2024-11-21T08:13:37,386 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/829529fdccf9853d96113fa67713ffbc 2024-11-21T08:13:37,387 INFO [StoreOpener-cd29f056dad8c6c37d0979434891505d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region cd29f056dad8c6c37d0979434891505d 2024-11-21T08:13:37,388 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/829529fdccf9853d96113fa67713ffbc 2024-11-21T08:13:37,389 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 829529fdccf9853d96113fa67713ffbc 2024-11-21T08:13:37,389 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 829529fdccf9853d96113fa67713ffbc 2024-11-21T08:13:37,389 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ef5fca53c8ff0d12fc112ab68731575, server=6ed56fb6cbf7,33091,1732176812968 because future has completed 2024-11-21T08:13:37,391 INFO [StoreOpener-cd29f056dad8c6c37d0979434891505d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cd29f056dad8c6c37d0979434891505d columnFamilyName family 2024-11-21T08:13:37,391 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 829529fdccf9853d96113fa67713ffbc 2024-11-21T08:13:37,391 DEBUG [StoreOpener-cd29f056dad8c6c37d0979434891505d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:37,391 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=99e7b3565f08f10e8c368072e9c21c09, regionState=OPEN, openSeqNum=2, regionLocation=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:37,392 INFO [StoreOpener-ebf8202a2e0e04a2c5c7f5f0e491d40e-1 {}] regionserver.HStore(327): Store=ebf8202a2e0e04a2c5c7f5f0e491d40e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:37,392 INFO [StoreOpener-cd29f056dad8c6c37d0979434891505d-1 {}] regionserver.HStore(327): Store=cd29f056dad8c6c37d0979434891505d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:37,393 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for ebf8202a2e0e04a2c5c7f5f0e491d40e 2024-11-21T08:13:37,393 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for cd29f056dad8c6c37d0979434891505d 2024-11-21T08:13:37,395 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/cd29f056dad8c6c37d0979434891505d 2024-11-21T08:13:37,395 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/ebf8202a2e0e04a2c5c7f5f0e491d40e 2024-11-21T08:13:37,395 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/cd29f056dad8c6c37d0979434891505d 2024-11-21T08:13:37,395 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/ebf8202a2e0e04a2c5c7f5f0e491d40e 2024-11-21T08:13:37,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 99e7b3565f08f10e8c368072e9c21c09, server=6ed56fb6cbf7,35295,1732176812677 because future has completed 2024-11-21T08:13:37,396 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for ebf8202a2e0e04a2c5c7f5f0e491d40e 2024-11-21T08:13:37,396 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/829529fdccf9853d96113fa67713ffbc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T08:13:37,396 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for ebf8202a2e0e04a2c5c7f5f0e491d40e 2024-11-21T08:13:37,397 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 829529fdccf9853d96113fa67713ffbc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70320363, jitterRate=0.047855064272880554}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:37,397 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 829529fdccf9853d96113fa67713ffbc 2024-11-21T08:13:37,398 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 829529fdccf9853d96113fa67713ffbc: Running coprocessor pre-open hook at 1732176817375Writing region info on filesystem at 1732176817375Initializing all the Stores at 1732176817377 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176817377Cleaning up temporary data from old regions at 1732176817389 (+12 ms)Running coprocessor post-open hooks at 1732176817397 (+8 ms)Region opened successfully at 1732176817398 (+1 ms) 2024-11-21T08:13:37,398 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for cd29f056dad8c6c37d0979434891505d 2024-11-21T08:13:37,398 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for cd29f056dad8c6c37d0979434891505d 2024-11-21T08:13:37,400 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1732176816148.829529fdccf9853d96113fa67713ffbc., pid=33, masterSystemTime=1732176817249 2024-11-21T08:13:37,402 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for ebf8202a2e0e04a2c5c7f5f0e491d40e 2024-11-21T08:13:37,403 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for cd29f056dad8c6c37d0979434891505d 2024-11-21T08:13:37,403 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1732176816148.829529fdccf9853d96113fa67713ffbc. 2024-11-21T08:13:37,403 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,fff,1732176816148.829529fdccf9853d96113fa67713ffbc. 2024-11-21T08:13:37,404 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,sss,1732176816148.4ed3111e35326fc2c5455ae2475ab1b5. 2024-11-21T08:13:37,404 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 4ed3111e35326fc2c5455ae2475ab1b5, NAME => 'test,sss,1732176816148.4ed3111e35326fc2c5455ae2475ab1b5.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-21T08:13:37,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=25 2024-11-21T08:13:37,404 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4ed3111e35326fc2c5455ae2475ab1b5 2024-11-21T08:13:37,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 7ef5fca53c8ff0d12fc112ab68731575, server=6ed56fb6cbf7,33091,1732176812968 in 205 msec 2024-11-21T08:13:37,405 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,sss,1732176816148.4ed3111e35326fc2c5455ae2475ab1b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:37,405 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 4ed3111e35326fc2c5455ae2475ab1b5 2024-11-21T08:13:37,405 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 4ed3111e35326fc2c5455ae2475ab1b5 2024-11-21T08:13:37,408 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, ASSIGN in 485 msec 2024-11-21T08:13:37,409 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/ebf8202a2e0e04a2c5c7f5f0e491d40e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T08:13:37,410 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened ebf8202a2e0e04a2c5c7f5f0e491d40e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71680806, jitterRate=0.06812724471092224}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:37,410 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ebf8202a2e0e04a2c5c7f5f0e491d40e 2024-11-21T08:13:37,410 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for ebf8202a2e0e04a2c5c7f5f0e491d40e: Running coprocessor pre-open hook at 1732176817378Writing region info on filesystem at 1732176817378Initializing all the Stores at 1732176817380 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176817380Cleaning up temporary data from old regions at 1732176817396 (+16 ms)Running coprocessor post-open hooks at 1732176817410 (+14 ms)Region opened successfully at 1732176817410 2024-11-21T08:13:37,411 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1732176816148.ebf8202a2e0e04a2c5c7f5f0e491d40e., pid=43, masterSystemTime=1732176817302 2024-11-21T08:13:37,412 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=829529fdccf9853d96113fa67713ffbc, regionState=OPEN, openSeqNum=2, regionLocation=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:37,414 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1732176816148.ebf8202a2e0e04a2c5c7f5f0e491d40e. 2024-11-21T08:13:37,415 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,www,1732176816148.ebf8202a2e0e04a2c5c7f5f0e491d40e. 2024-11-21T08:13:37,415 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 829529fdccf9853d96113fa67713ffbc, server=6ed56fb6cbf7,38845,1732176812866 because future has completed 2024-11-21T08:13:37,415 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,ddd,1732176816148.4f43187c46588bff73d01ba7eb26a6f8. 2024-11-21T08:13:37,415 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 4f43187c46588bff73d01ba7eb26a6f8, NAME => 'test,ddd,1732176816148.4f43187c46588bff73d01ba7eb26a6f8.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-21T08:13:37,416 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=ebf8202a2e0e04a2c5c7f5f0e491d40e, regionState=OPEN, openSeqNum=2, regionLocation=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:37,416 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4f43187c46588bff73d01ba7eb26a6f8 2024-11-21T08:13:37,416 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,ddd,1732176816148.4f43187c46588bff73d01ba7eb26a6f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:37,416 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 4f43187c46588bff73d01ba7eb26a6f8 2024-11-21T08:13:37,416 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 4f43187c46588bff73d01ba7eb26a6f8 2024-11-21T08:13:37,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure ebf8202a2e0e04a2c5c7f5f0e491d40e, server=6ed56fb6cbf7,35295,1732176812677 because future has completed 2024-11-21T08:13:37,420 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/cd29f056dad8c6c37d0979434891505d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T08:13:37,420 INFO [StoreOpener-4ed3111e35326fc2c5455ae2475ab1b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4ed3111e35326fc2c5455ae2475ab1b5 2024-11-21T08:13:37,421 INFO [StoreOpener-4f43187c46588bff73d01ba7eb26a6f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4f43187c46588bff73d01ba7eb26a6f8 2024-11-21T08:13:37,421 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened cd29f056dad8c6c37d0979434891505d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59878086, jitterRate=-0.10774698853492737}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:37,422 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cd29f056dad8c6c37d0979434891505d 2024-11-21T08:13:37,422 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for cd29f056dad8c6c37d0979434891505d: Running coprocessor pre-open hook at 1732176817385Writing region info on filesystem at 1732176817385Initializing all the Stores at 1732176817386 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176817386Cleaning up temporary data from old regions at 1732176817398 (+12 ms)Running coprocessor post-open hooks at 1732176817422 (+24 ms)Region opened successfully at 1732176817422 2024-11-21T08:13:37,424 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732176816148.cd29f056dad8c6c37d0979434891505d., pid=54, masterSystemTime=1732176817346 2024-11-21T08:13:37,424 INFO [StoreOpener-4f43187c46588bff73d01ba7eb26a6f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f43187c46588bff73d01ba7eb26a6f8 columnFamilyName family 2024-11-21T08:13:37,424 INFO [StoreOpener-4ed3111e35326fc2c5455ae2475ab1b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ed3111e35326fc2c5455ae2475ab1b5 columnFamilyName family 2024-11-21T08:13:37,424 DEBUG [StoreOpener-4f43187c46588bff73d01ba7eb26a6f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:37,424 DEBUG [StoreOpener-4ed3111e35326fc2c5455ae2475ab1b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:37,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=10 2024-11-21T08:13:37,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 829529fdccf9853d96113fa67713ffbc, server=6ed56fb6cbf7,38845,1732176812866 in 310 msec 2024-11-21T08:13:37,426 INFO [StoreOpener-4ed3111e35326fc2c5455ae2475ab1b5-1 {}] regionserver.HStore(327): Store=4ed3111e35326fc2c5455ae2475ab1b5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:37,427 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 4ed3111e35326fc2c5455ae2475ab1b5 2024-11-21T08:13:37,427 INFO [StoreOpener-4f43187c46588bff73d01ba7eb26a6f8-1 {}] regionserver.HStore(327): Store=4f43187c46588bff73d01ba7eb26a6f8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:37,427 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 4f43187c46588bff73d01ba7eb26a6f8 2024-11-21T08:13:37,428 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/4ed3111e35326fc2c5455ae2475ab1b5 2024-11-21T08:13:37,429 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/4f43187c46588bff73d01ba7eb26a6f8 2024-11-21T08:13:37,429 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732176816148.cd29f056dad8c6c37d0979434891505d. 2024-11-21T08:13:37,429 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,eee,1732176816148.cd29f056dad8c6c37d0979434891505d. 2024-11-21T08:13:37,429 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/4ed3111e35326fc2c5455ae2475ab1b5 2024-11-21T08:13:37,429 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/4f43187c46588bff73d01ba7eb26a6f8 2024-11-21T08:13:37,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=829529fdccf9853d96113fa67713ffbc, ASSIGN in 505 msec 2024-11-21T08:13:37,429 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,bbb,1732176816148.3773623806176f3d7f8efb3efe1b7937. 2024-11-21T08:13:37,430 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 3773623806176f3d7f8efb3efe1b7937, NAME => 'test,bbb,1732176816148.3773623806176f3d7f8efb3efe1b7937.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-21T08:13:37,430 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 4f43187c46588bff73d01ba7eb26a6f8 2024-11-21T08:13:37,430 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 4ed3111e35326fc2c5455ae2475ab1b5 2024-11-21T08:13:37,430 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 4f43187c46588bff73d01ba7eb26a6f8 2024-11-21T08:13:37,430 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 4ed3111e35326fc2c5455ae2475ab1b5 2024-11-21T08:13:37,430 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3773623806176f3d7f8efb3efe1b7937 2024-11-21T08:13:37,430 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,bbb,1732176816148.3773623806176f3d7f8efb3efe1b7937.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:37,431 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=cd29f056dad8c6c37d0979434891505d, regionState=OPEN, openSeqNum=2, regionLocation=6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:37,433 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 4f43187c46588bff73d01ba7eb26a6f8 2024-11-21T08:13:37,430 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 3773623806176f3d7f8efb3efe1b7937 2024-11-21T08:13:37,434 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 3773623806176f3d7f8efb3efe1b7937 2024-11-21T08:13:37,434 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 4ed3111e35326fc2c5455ae2475ab1b5 2024-11-21T08:13:37,435 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,33091,1732176812968 because future has completed 2024-11-21T08:13:37,437 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/4f43187c46588bff73d01ba7eb26a6f8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T08:13:37,439 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 4f43187c46588bff73d01ba7eb26a6f8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65709592, jitterRate=-0.020850777626037598}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:37,440 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4f43187c46588bff73d01ba7eb26a6f8 2024-11-21T08:13:37,440 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 4f43187c46588bff73d01ba7eb26a6f8: Running coprocessor pre-open hook at 1732176817416Writing region info on filesystem at 1732176817416Initializing all the Stores at 1732176817419 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176817419Cleaning up temporary data from old regions at 1732176817430 (+11 ms)Running coprocessor post-open hooks at 1732176817440 (+10 ms)Region opened successfully at 1732176817440 2024-11-21T08:13:37,440 INFO [StoreOpener-3773623806176f3d7f8efb3efe1b7937-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3773623806176f3d7f8efb3efe1b7937 2024-11-21T08:13:37,441 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1732176816148.4f43187c46588bff73d01ba7eb26a6f8., pid=48, masterSystemTime=1732176817302 2024-11-21T08:13:37,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=27 2024-11-21T08:13:37,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure ebf8202a2e0e04a2c5c7f5f0e491d40e, server=6ed56fb6cbf7,35295,1732176812677 in 275 msec 2024-11-21T08:13:37,444 INFO [StoreOpener-3773623806176f3d7f8efb3efe1b7937-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3773623806176f3d7f8efb3efe1b7937 columnFamilyName family 2024-11-21T08:13:37,444 DEBUG [StoreOpener-3773623806176f3d7f8efb3efe1b7937-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:37,444 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1732176816148.4f43187c46588bff73d01ba7eb26a6f8. 2024-11-21T08:13:37,444 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,ddd,1732176816148.4f43187c46588bff73d01ba7eb26a6f8. 2024-11-21T08:13:37,444 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,jjj,1732176816148.5fb158c8201cfbc5babdbfc33ee45ab1. 2024-11-21T08:13:37,445 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 5fb158c8201cfbc5babdbfc33ee45ab1, NAME => 'test,jjj,1732176816148.5fb158c8201cfbc5babdbfc33ee45ab1.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-21T08:13:37,445 INFO [StoreOpener-3773623806176f3d7f8efb3efe1b7937-1 {}] regionserver.HStore(327): Store=3773623806176f3d7f8efb3efe1b7937/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:37,445 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 3773623806176f3d7f8efb3efe1b7937 2024-11-21T08:13:37,445 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5fb158c8201cfbc5babdbfc33ee45ab1 2024-11-21T08:13:37,445 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,jjj,1732176816148.5fb158c8201cfbc5babdbfc33ee45ab1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:37,445 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 5fb158c8201cfbc5babdbfc33ee45ab1 2024-11-21T08:13:37,446 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 5fb158c8201cfbc5babdbfc33ee45ab1 2024-11-21T08:13:37,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ebf8202a2e0e04a2c5c7f5f0e491d40e, ASSIGN in 523 msec 2024-11-21T08:13:37,446 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/3773623806176f3d7f8efb3efe1b7937 2024-11-21T08:13:37,447 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/3773623806176f3d7f8efb3efe1b7937 2024-11-21T08:13:37,448 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 3773623806176f3d7f8efb3efe1b7937 2024-11-21T08:13:37,448 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 3773623806176f3d7f8efb3efe1b7937 2024-11-21T08:13:37,448 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=4f43187c46588bff73d01ba7eb26a6f8, regionState=OPEN, openSeqNum=2, regionLocation=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:37,449 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/4ed3111e35326fc2c5455ae2475ab1b5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T08:13:37,450 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 4ed3111e35326fc2c5455ae2475ab1b5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70583382, jitterRate=0.05177435278892517}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:37,451 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4ed3111e35326fc2c5455ae2475ab1b5 2024-11-21T08:13:37,451 INFO [StoreOpener-5fb158c8201cfbc5babdbfc33ee45ab1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5fb158c8201cfbc5babdbfc33ee45ab1 2024-11-21T08:13:37,451 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 4ed3111e35326fc2c5455ae2475ab1b5: Running coprocessor pre-open hook at 1732176817405Writing region info on filesystem at 1732176817405Initializing all the Stores at 1732176817407 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176817407Cleaning up temporary data from old regions at 1732176817430 (+23 ms)Running coprocessor post-open hooks at 1732176817451 (+21 ms)Region opened successfully at 1732176817451 2024-11-21T08:13:37,453 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f43187c46588bff73d01ba7eb26a6f8, server=6ed56fb6cbf7,35295,1732176812677 because future has completed 2024-11-21T08:13:37,453 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1732176816148.4ed3111e35326fc2c5455ae2475ab1b5., pid=37, masterSystemTime=1732176817249 2024-11-21T08:13:37,454 INFO [StoreOpener-5fb158c8201cfbc5babdbfc33ee45ab1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5fb158c8201cfbc5babdbfc33ee45ab1 columnFamilyName family 2024-11-21T08:13:37,454 DEBUG [StoreOpener-5fb158c8201cfbc5babdbfc33ee45ab1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:37,455 INFO [StoreOpener-5fb158c8201cfbc5babdbfc33ee45ab1-1 {}] regionserver.HStore(327): Store=5fb158c8201cfbc5babdbfc33ee45ab1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:37,455 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 5fb158c8201cfbc5babdbfc33ee45ab1 2024-11-21T08:13:37,456 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 3773623806176f3d7f8efb3efe1b7937 2024-11-21T08:13:37,457 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/5fb158c8201cfbc5babdbfc33ee45ab1 2024-11-21T08:13:37,458 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/5fb158c8201cfbc5babdbfc33ee45ab1 2024-11-21T08:13:37,458 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1732176816148.4ed3111e35326fc2c5455ae2475ab1b5. 2024-11-21T08:13:37,458 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,sss,1732176816148.4ed3111e35326fc2c5455ae2475ab1b5. 2024-11-21T08:13:37,458 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 5fb158c8201cfbc5babdbfc33ee45ab1 2024-11-21T08:13:37,458 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 5fb158c8201cfbc5babdbfc33ee45ab1 2024-11-21T08:13:37,459 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,yyy,1732176816148.af43494f9c57dd49b6d504756610f96c. 2024-11-21T08:13:37,459 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => af43494f9c57dd49b6d504756610f96c, NAME => 'test,yyy,1732176816148.af43494f9c57dd49b6d504756610f96c.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-21T08:13:37,460 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test af43494f9c57dd49b6d504756610f96c 2024-11-21T08:13:37,460 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,yyy,1732176816148.af43494f9c57dd49b6d504756610f96c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:37,460 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for af43494f9c57dd49b6d504756610f96c 2024-11-21T08:13:37,460 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for af43494f9c57dd49b6d504756610f96c 2024-11-21T08:13:37,462 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 5fb158c8201cfbc5babdbfc33ee45ab1 2024-11-21T08:13:37,463 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=9 2024-11-21T08:13:37,463 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/3773623806176f3d7f8efb3efe1b7937/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T08:13:37,463 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,33091,1732176812968 in 211 msec 2024-11-21T08:13:37,463 INFO [StoreOpener-af43494f9c57dd49b6d504756610f96c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region af43494f9c57dd49b6d504756610f96c 2024-11-21T08:13:37,464 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 3773623806176f3d7f8efb3efe1b7937; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63333451, jitterRate=-0.05625803768634796}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:37,464 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3773623806176f3d7f8efb3efe1b7937 2024-11-21T08:13:37,464 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 3773623806176f3d7f8efb3efe1b7937: Running coprocessor pre-open hook at 1732176817434Writing region info on filesystem at 1732176817434Initializing all the Stores at 1732176817437 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176817437Cleaning up temporary data from old regions at 1732176817448 (+11 ms)Running coprocessor post-open hooks at 1732176817464 (+16 ms)Region opened successfully at 1732176817464 2024-11-21T08:13:37,465 INFO [StoreOpener-af43494f9c57dd49b6d504756610f96c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region af43494f9c57dd49b6d504756610f96c columnFamilyName family 2024-11-21T08:13:37,466 DEBUG [StoreOpener-af43494f9c57dd49b6d504756610f96c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:37,466 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1732176816148.3773623806176f3d7f8efb3efe1b7937., pid=56, masterSystemTime=1732176817346 2024-11-21T08:13:37,466 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/5fb158c8201cfbc5babdbfc33ee45ab1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T08:13:37,467 INFO [StoreOpener-af43494f9c57dd49b6d504756610f96c-1 {}] regionserver.HStore(327): Store=af43494f9c57dd49b6d504756610f96c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:37,467 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for af43494f9c57dd49b6d504756610f96c 2024-11-21T08:13:37,467 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 5fb158c8201cfbc5babdbfc33ee45ab1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67523620, jitterRate=0.0061803460121154785}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:37,467 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5fb158c8201cfbc5babdbfc33ee45ab1 2024-11-21T08:13:37,468 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/af43494f9c57dd49b6d504756610f96c 2024-11-21T08:13:37,468 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 5fb158c8201cfbc5babdbfc33ee45ab1: Running coprocessor pre-open hook at 1732176817446Writing region info on filesystem at 1732176817446Initializing all the Stores at 1732176817447 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176817448 (+1 ms)Cleaning up temporary data from old regions at 1732176817458 (+10 ms)Running coprocessor post-open hooks at 1732176817468 (+10 ms)Region opened successfully at 1732176817468 2024-11-21T08:13:37,469 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/af43494f9c57dd49b6d504756610f96c 2024-11-21T08:13:37,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN in 543 msec 2024-11-21T08:13:37,470 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1732176816148.5fb158c8201cfbc5babdbfc33ee45ab1., pid=45, masterSystemTime=1732176817302 2024-11-21T08:13:37,470 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for af43494f9c57dd49b6d504756610f96c 2024-11-21T08:13:37,470 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for af43494f9c57dd49b6d504756610f96c 2024-11-21T08:13:37,470 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1732176816148.3773623806176f3d7f8efb3efe1b7937. 2024-11-21T08:13:37,471 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,bbb,1732176816148.3773623806176f3d7f8efb3efe1b7937. 2024-11-21T08:13:37,471 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,xxx,1732176816148.021eaa39c912c55ccf9b758a5c92d1f8. 2024-11-21T08:13:37,472 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=4ed3111e35326fc2c5455ae2475ab1b5, regionState=OPEN, openSeqNum=2, regionLocation=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:37,471 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 021eaa39c912c55ccf9b758a5c92d1f8, NAME => 'test,xxx,1732176816148.021eaa39c912c55ccf9b758a5c92d1f8.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-21T08:13:37,473 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 021eaa39c912c55ccf9b758a5c92d1f8 2024-11-21T08:13:37,473 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,xxx,1732176816148.021eaa39c912c55ccf9b758a5c92d1f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:37,473 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for af43494f9c57dd49b6d504756610f96c 2024-11-21T08:13:37,473 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 021eaa39c912c55ccf9b758a5c92d1f8 2024-11-21T08:13:37,473 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 021eaa39c912c55ccf9b758a5c92d1f8 2024-11-21T08:13:37,473 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1732176816148.5fb158c8201cfbc5babdbfc33ee45ab1. 2024-11-21T08:13:37,473 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,jjj,1732176816148.5fb158c8201cfbc5babdbfc33ee45ab1. 2024-11-21T08:13:37,473 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,qqq,1732176816148.2d5b1f002057972e76beae8f19094030. 2024-11-21T08:13:37,474 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 2d5b1f002057972e76beae8f19094030, NAME => 'test,qqq,1732176816148.2d5b1f002057972e76beae8f19094030.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-21T08:13:37,474 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2d5b1f002057972e76beae8f19094030 2024-11-21T08:13:37,475 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,qqq,1732176816148.2d5b1f002057972e76beae8f19094030.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:37,475 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 2d5b1f002057972e76beae8f19094030 2024-11-21T08:13:37,475 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 2d5b1f002057972e76beae8f19094030 2024-11-21T08:13:37,476 INFO [StoreOpener-021eaa39c912c55ccf9b758a5c92d1f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 021eaa39c912c55ccf9b758a5c92d1f8 2024-11-21T08:13:37,479 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=6ed56fb6cbf7,38845,1732176812866, table=test, region=4ed3111e35326fc2c5455ae2475ab1b5. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T08:13:37,479 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/af43494f9c57dd49b6d504756610f96c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T08:13:37,480 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4ed3111e35326fc2c5455ae2475ab1b5, server=6ed56fb6cbf7,38845,1732176812866 because future has completed 2024-11-21T08:13:37,480 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened af43494f9c57dd49b6d504756610f96c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62975935, jitterRate=-0.0615854412317276}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:37,480 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for af43494f9c57dd49b6d504756610f96c 2024-11-21T08:13:37,480 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for af43494f9c57dd49b6d504756610f96c: Running coprocessor pre-open hook at 1732176817460Writing region info on filesystem at 1732176817460Initializing all the Stores at 1732176817462 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176817462Cleaning up temporary data from old regions at 1732176817470 (+8 ms)Running coprocessor post-open hooks at 1732176817480 (+10 ms)Region opened successfully at 1732176817480 2024-11-21T08:13:37,481 INFO [StoreOpener-021eaa39c912c55ccf9b758a5c92d1f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 021eaa39c912c55ccf9b758a5c92d1f8 columnFamilyName family 2024-11-21T08:13:37,481 DEBUG [StoreOpener-021eaa39c912c55ccf9b758a5c92d1f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:37,481 INFO [StoreOpener-2d5b1f002057972e76beae8f19094030-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2d5b1f002057972e76beae8f19094030 2024-11-21T08:13:37,482 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1732176816148.af43494f9c57dd49b6d504756610f96c., pid=31, masterSystemTime=1732176817249 2024-11-21T08:13:37,482 INFO [StoreOpener-021eaa39c912c55ccf9b758a5c92d1f8-1 {}] regionserver.HStore(327): Store=021eaa39c912c55ccf9b758a5c92d1f8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:37,482 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=test, region=4f43187c46588bff73d01ba7eb26a6f8. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T08:13:37,482 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=test, region=99e7b3565f08f10e8c368072e9c21c09. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T08:13:37,482 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 021eaa39c912c55ccf9b758a5c92d1f8 2024-11-21T08:13:37,485 INFO [StoreOpener-2d5b1f002057972e76beae8f19094030-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d5b1f002057972e76beae8f19094030 columnFamilyName family 2024-11-21T08:13:37,485 DEBUG [StoreOpener-2d5b1f002057972e76beae8f19094030-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:37,485 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/021eaa39c912c55ccf9b758a5c92d1f8 2024-11-21T08:13:37,486 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1732176816148.af43494f9c57dd49b6d504756610f96c. 2024-11-21T08:13:37,486 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/021eaa39c912c55ccf9b758a5c92d1f8 2024-11-21T08:13:37,486 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,yyy,1732176816148.af43494f9c57dd49b6d504756610f96c. 2024-11-21T08:13:37,486 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,ppp,1732176816148.5017ae3acb74fe8c7d0b9c6d9a2cc402. 2024-11-21T08:13:37,486 INFO [StoreOpener-2d5b1f002057972e76beae8f19094030-1 {}] regionserver.HStore(327): Store=2d5b1f002057972e76beae8f19094030/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:37,486 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 5017ae3acb74fe8c7d0b9c6d9a2cc402, NAME => 'test,ppp,1732176816148.5017ae3acb74fe8c7d0b9c6d9a2cc402.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-21T08:13:37,486 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 2d5b1f002057972e76beae8f19094030 2024-11-21T08:13:37,487 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5017ae3acb74fe8c7d0b9c6d9a2cc402 2024-11-21T08:13:37,487 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,ppp,1732176816148.5017ae3acb74fe8c7d0b9c6d9a2cc402.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:37,487 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 5017ae3acb74fe8c7d0b9c6d9a2cc402 2024-11-21T08:13:37,487 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 5017ae3acb74fe8c7d0b9c6d9a2cc402 2024-11-21T08:13:37,487 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 021eaa39c912c55ccf9b758a5c92d1f8 2024-11-21T08:13:37,487 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 021eaa39c912c55ccf9b758a5c92d1f8 2024-11-21T08:13:37,487 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/2d5b1f002057972e76beae8f19094030 2024-11-21T08:13:37,488 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/2d5b1f002057972e76beae8f19094030 2024-11-21T08:13:37,489 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 2d5b1f002057972e76beae8f19094030 2024-11-21T08:13:37,489 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 2d5b1f002057972e76beae8f19094030 2024-11-21T08:13:37,490 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 021eaa39c912c55ccf9b758a5c92d1f8 2024-11-21T08:13:37,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=23 2024-11-21T08:13:37,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 4ed3111e35326fc2c5455ae2475ab1b5, server=6ed56fb6cbf7,38845,1732176812866 in 354 msec 2024-11-21T08:13:37,492 INFO [StoreOpener-5017ae3acb74fe8c7d0b9c6d9a2cc402-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5017ae3acb74fe8c7d0b9c6d9a2cc402 2024-11-21T08:13:37,493 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 2d5b1f002057972e76beae8f19094030 2024-11-21T08:13:37,494 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/021eaa39c912c55ccf9b758a5c92d1f8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T08:13:37,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4ed3111e35326fc2c5455ae2475ab1b5, ASSIGN in 572 msec 2024-11-21T08:13:37,496 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 021eaa39c912c55ccf9b758a5c92d1f8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72644105, jitterRate=0.0824815183877945}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:37,496 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 021eaa39c912c55ccf9b758a5c92d1f8 2024-11-21T08:13:37,496 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 021eaa39c912c55ccf9b758a5c92d1f8: Running coprocessor pre-open hook at 1732176817473Writing region info on filesystem at 1732176817473Initializing all the Stores at 1732176817476 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176817476Cleaning up temporary data from old regions at 1732176817487 (+11 ms)Running coprocessor post-open hooks at 1732176817496 (+9 ms)Region opened successfully at 1732176817496 2024-11-21T08:13:37,496 INFO [StoreOpener-5017ae3acb74fe8c7d0b9c6d9a2cc402-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5017ae3acb74fe8c7d0b9c6d9a2cc402 columnFamilyName family 2024-11-21T08:13:37,497 DEBUG [StoreOpener-5017ae3acb74fe8c7d0b9c6d9a2cc402-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:37,497 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/2d5b1f002057972e76beae8f19094030/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T08:13:37,498 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=af43494f9c57dd49b6d504756610f96c, regionState=OPEN, openSeqNum=2, regionLocation=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:37,498 INFO [StoreOpener-5017ae3acb74fe8c7d0b9c6d9a2cc402-1 {}] regionserver.HStore(327): Store=5017ae3acb74fe8c7d0b9c6d9a2cc402/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:37,498 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732176816148.021eaa39c912c55ccf9b758a5c92d1f8., pid=47, masterSystemTime=1732176817346 2024-11-21T08:13:37,498 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 5017ae3acb74fe8c7d0b9c6d9a2cc402 2024-11-21T08:13:37,498 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 2d5b1f002057972e76beae8f19094030; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71064475, jitterRate=0.05894319713115692}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:37,498 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2d5b1f002057972e76beae8f19094030 2024-11-21T08:13:37,499 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 2d5b1f002057972e76beae8f19094030: Running coprocessor pre-open hook at 1732176817476Writing region info on filesystem at 1732176817476Initializing all the Stores at 1732176817477 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176817477Cleaning up temporary data from old regions at 1732176817489 (+12 ms)Running coprocessor post-open hooks at 1732176817498 (+9 ms)Region opened successfully at 1732176817498 2024-11-21T08:13:37,499 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/5017ae3acb74fe8c7d0b9c6d9a2cc402 2024-11-21T08:13:37,501 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/5017ae3acb74fe8c7d0b9c6d9a2cc402 2024-11-21T08:13:37,503 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure af43494f9c57dd49b6d504756610f96c, server=6ed56fb6cbf7,38845,1732176812866 because future has completed 2024-11-21T08:13:37,503 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1732176816148.2d5b1f002057972e76beae8f19094030., pid=42, masterSystemTime=1732176817302 2024-11-21T08:13:37,504 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 5017ae3acb74fe8c7d0b9c6d9a2cc402 2024-11-21T08:13:37,504 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 5017ae3acb74fe8c7d0b9c6d9a2cc402 2024-11-21T08:13:37,504 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732176816148.021eaa39c912c55ccf9b758a5c92d1f8. 2024-11-21T08:13:37,504 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,xxx,1732176816148.021eaa39c912c55ccf9b758a5c92d1f8. 2024-11-21T08:13:37,505 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,ooo,1732176816148.477e65b3290a1e6e03e5eb59f7aa956d. 2024-11-21T08:13:37,505 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 477e65b3290a1e6e03e5eb59f7aa956d, NAME => 'test,ooo,1732176816148.477e65b3290a1e6e03e5eb59f7aa956d.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-21T08:13:37,505 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 477e65b3290a1e6e03e5eb59f7aa956d 2024-11-21T08:13:37,505 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,ooo,1732176816148.477e65b3290a1e6e03e5eb59f7aa956d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:37,505 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 477e65b3290a1e6e03e5eb59f7aa956d 2024-11-21T08:13:37,505 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 477e65b3290a1e6e03e5eb59f7aa956d 2024-11-21T08:13:37,508 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1732176816148.2d5b1f002057972e76beae8f19094030. 2024-11-21T08:13:37,508 INFO [StoreOpener-477e65b3290a1e6e03e5eb59f7aa956d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 477e65b3290a1e6e03e5eb59f7aa956d 2024-11-21T08:13:37,508 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,qqq,1732176816148.2d5b1f002057972e76beae8f19094030. 2024-11-21T08:13:37,508 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,ggg,1732176816148.ee7ef14c3df8915e550cfda528563489. 2024-11-21T08:13:37,508 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 5017ae3acb74fe8c7d0b9c6d9a2cc402 2024-11-21T08:13:37,508 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => ee7ef14c3df8915e550cfda528563489, NAME => 'test,ggg,1732176816148.ee7ef14c3df8915e550cfda528563489.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-21T08:13:37,509 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ee7ef14c3df8915e550cfda528563489 2024-11-21T08:13:37,509 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,ggg,1732176816148.ee7ef14c3df8915e550cfda528563489.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:37,509 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for ee7ef14c3df8915e550cfda528563489 2024-11-21T08:13:37,509 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for ee7ef14c3df8915e550cfda528563489 2024-11-21T08:13:37,512 INFO [StoreOpener-477e65b3290a1e6e03e5eb59f7aa956d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 477e65b3290a1e6e03e5eb59f7aa956d columnFamilyName family 2024-11-21T08:13:37,513 DEBUG [StoreOpener-477e65b3290a1e6e03e5eb59f7aa956d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:37,513 INFO [StoreOpener-ee7ef14c3df8915e550cfda528563489-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ee7ef14c3df8915e550cfda528563489 2024-11-21T08:13:37,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=29 2024-11-21T08:13:37,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure af43494f9c57dd49b6d504756610f96c, server=6ed56fb6cbf7,38845,1732176812866 in 411 msec 2024-11-21T08:13:37,513 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/5017ae3acb74fe8c7d0b9c6d9a2cc402/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T08:13:37,514 INFO [StoreOpener-477e65b3290a1e6e03e5eb59f7aa956d-1 {}] regionserver.HStore(327): Store=477e65b3290a1e6e03e5eb59f7aa956d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:37,514 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 477e65b3290a1e6e03e5eb59f7aa956d 2024-11-21T08:13:37,514 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 5017ae3acb74fe8c7d0b9c6d9a2cc402; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64719095, jitterRate=-0.03561033308506012}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:37,514 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5017ae3acb74fe8c7d0b9c6d9a2cc402 2024-11-21T08:13:37,515 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 5017ae3acb74fe8c7d0b9c6d9a2cc402: Running coprocessor pre-open hook at 1732176817487Writing region info on filesystem at 1732176817487Initializing all the Stores at 1732176817489 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176817489Cleaning up temporary data from old regions at 1732176817504 (+15 ms)Running coprocessor post-open hooks at 1732176817514 (+10 ms)Region opened successfully at 1732176817515 (+1 ms) 2024-11-21T08:13:37,516 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/477e65b3290a1e6e03e5eb59f7aa956d 2024-11-21T08:13:37,516 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1732176816148.5017ae3acb74fe8c7d0b9c6d9a2cc402., pid=36, masterSystemTime=1732176817249 2024-11-21T08:13:37,516 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/477e65b3290a1e6e03e5eb59f7aa956d 2024-11-21T08:13:37,516 INFO [StoreOpener-ee7ef14c3df8915e550cfda528563489-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee7ef14c3df8915e550cfda528563489 columnFamilyName family 2024-11-21T08:13:37,517 DEBUG [StoreOpener-ee7ef14c3df8915e550cfda528563489-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:37,517 INFO [StoreOpener-ee7ef14c3df8915e550cfda528563489-1 {}] regionserver.HStore(327): Store=ee7ef14c3df8915e550cfda528563489/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:37,518 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for ee7ef14c3df8915e550cfda528563489 2024-11-21T08:13:37,518 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 477e65b3290a1e6e03e5eb59f7aa956d 2024-11-21T08:13:37,518 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 477e65b3290a1e6e03e5eb59f7aa956d 2024-11-21T08:13:37,519 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/ee7ef14c3df8915e550cfda528563489 2024-11-21T08:13:37,520 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1732176816148.5017ae3acb74fe8c7d0b9c6d9a2cc402. 2024-11-21T08:13:37,520 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/ee7ef14c3df8915e550cfda528563489 2024-11-21T08:13:37,520 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,ppp,1732176816148.5017ae3acb74fe8c7d0b9c6d9a2cc402. 2024-11-21T08:13:37,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=af43494f9c57dd49b6d504756610f96c, ASSIGN in 593 msec 2024-11-21T08:13:37,520 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,ccc,1732176816148.f38fd8b8defc002b7cb3de558717f27c. 2024-11-21T08:13:37,520 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => f38fd8b8defc002b7cb3de558717f27c, NAME => 'test,ccc,1732176816148.f38fd8b8defc002b7cb3de558717f27c.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-21T08:13:37,520 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for ee7ef14c3df8915e550cfda528563489 2024-11-21T08:13:37,520 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f38fd8b8defc002b7cb3de558717f27c 2024-11-21T08:13:37,520 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for ee7ef14c3df8915e550cfda528563489 2024-11-21T08:13:37,521 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,ccc,1732176816148.f38fd8b8defc002b7cb3de558717f27c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:37,521 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for f38fd8b8defc002b7cb3de558717f27c 2024-11-21T08:13:37,521 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for f38fd8b8defc002b7cb3de558717f27c 2024-11-21T08:13:37,522 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=5017ae3acb74fe8c7d0b9c6d9a2cc402, regionState=OPEN, openSeqNum=2, regionLocation=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:37,522 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 477e65b3290a1e6e03e5eb59f7aa956d 2024-11-21T08:13:37,523 INFO [StoreOpener-f38fd8b8defc002b7cb3de558717f27c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f38fd8b8defc002b7cb3de558717f27c 2024-11-21T08:13:37,523 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for ee7ef14c3df8915e550cfda528563489 2024-11-21T08:13:37,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5017ae3acb74fe8c7d0b9c6d9a2cc402, server=6ed56fb6cbf7,38845,1732176812866 because future has completed 2024-11-21T08:13:37,527 INFO [StoreOpener-f38fd8b8defc002b7cb3de558717f27c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f38fd8b8defc002b7cb3de558717f27c columnFamilyName family 2024-11-21T08:13:37,527 DEBUG [StoreOpener-f38fd8b8defc002b7cb3de558717f27c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:37,528 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=2d5b1f002057972e76beae8f19094030, regionState=OPEN, openSeqNum=2, regionLocation=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:37,528 INFO [StoreOpener-f38fd8b8defc002b7cb3de558717f27c-1 {}] regionserver.HStore(327): Store=f38fd8b8defc002b7cb3de558717f27c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:37,529 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for f38fd8b8defc002b7cb3de558717f27c 2024-11-21T08:13:37,529 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/477e65b3290a1e6e03e5eb59f7aa956d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T08:13:37,530 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/f38fd8b8defc002b7cb3de558717f27c 2024-11-21T08:13:37,531 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/f38fd8b8defc002b7cb3de558717f27c 2024-11-21T08:13:37,531 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 477e65b3290a1e6e03e5eb59f7aa956d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68216522, jitterRate=0.016505390405654907}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:37,532 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 477e65b3290a1e6e03e5eb59f7aa956d 2024-11-21T08:13:37,532 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d5b1f002057972e76beae8f19094030, server=6ed56fb6cbf7,35295,1732176812677 because future has completed 2024-11-21T08:13:37,532 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for f38fd8b8defc002b7cb3de558717f27c 2024-11-21T08:13:37,532 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for f38fd8b8defc002b7cb3de558717f27c 2024-11-21T08:13:37,533 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 477e65b3290a1e6e03e5eb59f7aa956d: Running coprocessor pre-open hook at 1732176817506Writing region info on filesystem at 1732176817506Initializing all the Stores at 1732176817507 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176817507Cleaning up temporary data from old regions at 1732176817518 (+11 ms)Running coprocessor post-open hooks at 1732176817532 (+14 ms)Region opened successfully at 1732176817532 2024-11-21T08:13:37,534 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1732176816148.477e65b3290a1e6e03e5eb59f7aa956d., pid=53, masterSystemTime=1732176817346 2024-11-21T08:13:37,534 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/ee7ef14c3df8915e550cfda528563489/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T08:13:37,535 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened ee7ef14c3df8915e550cfda528563489; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66288108, jitterRate=-0.012230217456817627}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:37,535 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ee7ef14c3df8915e550cfda528563489 2024-11-21T08:13:37,535 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for ee7ef14c3df8915e550cfda528563489: Running coprocessor pre-open hook at 1732176817510Writing region info on filesystem at 1732176817510Initializing all the Stores at 1732176817511 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176817511Cleaning up temporary data from old regions at 1732176817520 (+9 ms)Running coprocessor post-open hooks at 1732176817535 (+15 ms)Region opened successfully at 1732176817535 2024-11-21T08:13:37,536 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for f38fd8b8defc002b7cb3de558717f27c 2024-11-21T08:13:37,536 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1732176816148.ee7ef14c3df8915e550cfda528563489., pid=49, masterSystemTime=1732176817302 2024-11-21T08:13:37,538 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1732176816148.477e65b3290a1e6e03e5eb59f7aa956d. 2024-11-21T08:13:37,538 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,ooo,1732176816148.477e65b3290a1e6e03e5eb59f7aa956d. 2024-11-21T08:13:37,538 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,hhh,1732176816148.9eb0364d53b384c3e59ecc15348e6c3c. 2024-11-21T08:13:37,539 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=20 2024-11-21T08:13:37,539 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 9eb0364d53b384c3e59ecc15348e6c3c, NAME => 'test,hhh,1732176816148.9eb0364d53b384c3e59ecc15348e6c3c.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-21T08:13:37,539 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 5017ae3acb74fe8c7d0b9c6d9a2cc402, server=6ed56fb6cbf7,38845,1732176812866 in 407 msec 2024-11-21T08:13:37,539 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9eb0364d53b384c3e59ecc15348e6c3c 2024-11-21T08:13:37,539 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,hhh,1732176816148.9eb0364d53b384c3e59ecc15348e6c3c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:37,539 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 9eb0364d53b384c3e59ecc15348e6c3c 2024-11-21T08:13:37,540 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 9eb0364d53b384c3e59ecc15348e6c3c 2024-11-21T08:13:37,540 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1732176816148.ee7ef14c3df8915e550cfda528563489. 2024-11-21T08:13:37,540 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,ggg,1732176816148.ee7ef14c3df8915e550cfda528563489. 2024-11-21T08:13:37,540 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,ttt,1732176816148.ec3a5c93577675c61d946ed41b85bdb5. 2024-11-21T08:13:37,540 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => ec3a5c93577675c61d946ed41b85bdb5, NAME => 'test,ttt,1732176816148.ec3a5c93577675c61d946ed41b85bdb5.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-21T08:13:37,541 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ec3a5c93577675c61d946ed41b85bdb5 2024-11-21T08:13:37,541 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,ttt,1732176816148.ec3a5c93577675c61d946ed41b85bdb5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:37,541 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for ec3a5c93577675c61d946ed41b85bdb5 2024-11-21T08:13:37,541 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for ec3a5c93577675c61d946ed41b85bdb5 2024-11-21T08:13:37,541 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/f38fd8b8defc002b7cb3de558717f27c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T08:13:37,542 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened f38fd8b8defc002b7cb3de558717f27c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61956195, jitterRate=-0.07678075134754181}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:37,542 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f38fd8b8defc002b7cb3de558717f27c 2024-11-21T08:13:37,543 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for f38fd8b8defc002b7cb3de558717f27c: Running coprocessor pre-open hook at 1732176817521Writing region info on filesystem at 1732176817521Initializing all the Stores at 1732176817522 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176817523 (+1 ms)Cleaning up temporary data from old regions at 1732176817532 (+9 ms)Running coprocessor post-open hooks at 1732176817542 (+10 ms)Region opened successfully at 1732176817542 2024-11-21T08:13:37,543 INFO [StoreOpener-9eb0364d53b384c3e59ecc15348e6c3c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9eb0364d53b384c3e59ecc15348e6c3c 2024-11-21T08:13:37,543 INFO [StoreOpener-ec3a5c93577675c61d946ed41b85bdb5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ec3a5c93577675c61d946ed41b85bdb5 2024-11-21T08:13:37,544 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5017ae3acb74fe8c7d0b9c6d9a2cc402, ASSIGN in 619 msec 2024-11-21T08:13:37,544 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732176816148.f38fd8b8defc002b7cb3de558717f27c., pid=39, masterSystemTime=1732176817249 2024-11-21T08:13:37,546 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=ee7ef14c3df8915e550cfda528563489, regionState=OPEN, openSeqNum=2, regionLocation=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:37,547 INFO [StoreOpener-ec3a5c93577675c61d946ed41b85bdb5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec3a5c93577675c61d946ed41b85bdb5 columnFamilyName family 2024-11-21T08:13:37,547 INFO [StoreOpener-9eb0364d53b384c3e59ecc15348e6c3c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9eb0364d53b384c3e59ecc15348e6c3c columnFamilyName family 2024-11-21T08:13:37,547 DEBUG [StoreOpener-9eb0364d53b384c3e59ecc15348e6c3c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:37,547 DEBUG [StoreOpener-ec3a5c93577675c61d946ed41b85bdb5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:37,547 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1732176816148.f38fd8b8defc002b7cb3de558717f27c. 2024-11-21T08:13:37,548 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,ccc,1732176816148.f38fd8b8defc002b7cb3de558717f27c. 2024-11-21T08:13:37,548 INFO [StoreOpener-ec3a5c93577675c61d946ed41b85bdb5-1 {}] regionserver.HStore(327): Store=ec3a5c93577675c61d946ed41b85bdb5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:37,549 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for ec3a5c93577675c61d946ed41b85bdb5 2024-11-21T08:13:37,549 INFO [StoreOpener-9eb0364d53b384c3e59ecc15348e6c3c-1 {}] regionserver.HStore(327): Store=9eb0364d53b384c3e59ecc15348e6c3c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:37,549 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 9eb0364d53b384c3e59ecc15348e6c3c 2024-11-21T08:13:37,550 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/ec3a5c93577675c61d946ed41b85bdb5 2024-11-21T08:13:37,550 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee7ef14c3df8915e550cfda528563489, server=6ed56fb6cbf7,35295,1732176812677 because future has completed 2024-11-21T08:13:37,551 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/ec3a5c93577675c61d946ed41b85bdb5 2024-11-21T08:13:37,551 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/9eb0364d53b384c3e59ecc15348e6c3c 2024-11-21T08:13:37,551 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/9eb0364d53b384c3e59ecc15348e6c3c 2024-11-21T08:13:37,551 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f38fd8b8defc002b7cb3de558717f27c, regionState=OPEN, openSeqNum=2, regionLocation=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:37,551 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for ec3a5c93577675c61d946ed41b85bdb5 2024-11-21T08:13:37,551 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for ec3a5c93577675c61d946ed41b85bdb5 2024-11-21T08:13:37,552 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 9eb0364d53b384c3e59ecc15348e6c3c 2024-11-21T08:13:37,552 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 9eb0364d53b384c3e59ecc15348e6c3c 2024-11-21T08:13:37,554 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for ec3a5c93577675c61d946ed41b85bdb5 2024-11-21T08:13:37,554 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 9eb0364d53b384c3e59ecc15348e6c3c 2024-11-21T08:13:37,555 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure f38fd8b8defc002b7cb3de558717f27c, server=6ed56fb6cbf7,38845,1732176812866 because future has completed 2024-11-21T08:13:37,558 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/ec3a5c93577675c61d946ed41b85bdb5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T08:13:37,559 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened ec3a5c93577675c61d946ed41b85bdb5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61716340, jitterRate=-0.08035486936569214}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:37,559 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ec3a5c93577675c61d946ed41b85bdb5 2024-11-21T08:13:37,559 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for ec3a5c93577675c61d946ed41b85bdb5: Running coprocessor pre-open hook at 1732176817541Writing region info on filesystem at 1732176817541Initializing all the Stores at 1732176817543 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176817543Cleaning up temporary data from old regions at 1732176817551 (+8 ms)Running coprocessor post-open hooks at 1732176817559 (+8 ms)Region opened successfully at 1732176817559 2024-11-21T08:13:37,560 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1732176816148.ec3a5c93577675c61d946ed41b85bdb5., pid=40, masterSystemTime=1732176817302 2024-11-21T08:13:37,561 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/9eb0364d53b384c3e59ecc15348e6c3c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T08:13:37,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=11 2024-11-21T08:13:37,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure ee7ef14c3df8915e550cfda528563489, server=6ed56fb6cbf7,35295,1732176812677 in 344 msec 2024-11-21T08:13:37,562 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 9eb0364d53b384c3e59ecc15348e6c3c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64063019, jitterRate=-0.045386627316474915}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:37,562 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9eb0364d53b384c3e59ecc15348e6c3c 2024-11-21T08:13:37,562 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 9eb0364d53b384c3e59ecc15348e6c3c: Running coprocessor pre-open hook at 1732176817540Writing region info on filesystem at 1732176817540Initializing all the Stores at 1732176817542 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176817542Cleaning up temporary data from old regions at 1732176817552 (+10 ms)Running coprocessor post-open hooks at 1732176817562 (+10 ms)Region opened successfully at 1732176817562 2024-11-21T08:13:37,563 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1732176816148.ec3a5c93577675c61d946ed41b85bdb5. 2024-11-21T08:13:37,563 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,ttt,1732176816148.ec3a5c93577675c61d946ed41b85bdb5. 2024-11-21T08:13:37,563 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1732176816148.9eb0364d53b384c3e59ecc15348e6c3c., pid=51, masterSystemTime=1732176817346 2024-11-21T08:13:37,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ee7ef14c3df8915e550cfda528563489, ASSIGN in 641 msec 2024-11-21T08:13:37,566 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1732176816148.9eb0364d53b384c3e59ecc15348e6c3c. 2024-11-21T08:13:37,566 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,hhh,1732176816148.9eb0364d53b384c3e59ecc15348e6c3c. 2024-11-21T08:13:37,566 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=ec3a5c93577675c61d946ed41b85bdb5, regionState=OPEN, openSeqNum=2, regionLocation=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:37,566 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,kkk,1732176816148.dbfd5210cf46b2a246edeeebd244b662. 2024-11-21T08:13:37,566 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => dbfd5210cf46b2a246edeeebd244b662, NAME => 'test,kkk,1732176816148.dbfd5210cf46b2a246edeeebd244b662.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-21T08:13:37,567 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test dbfd5210cf46b2a246edeeebd244b662 2024-11-21T08:13:37,567 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,kkk,1732176816148.dbfd5210cf46b2a246edeeebd244b662.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:37,567 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for dbfd5210cf46b2a246edeeebd244b662 2024-11-21T08:13:37,567 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for dbfd5210cf46b2a246edeeebd244b662 2024-11-21T08:13:37,570 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure ec3a5c93577675c61d946ed41b85bdb5, server=6ed56fb6cbf7,35295,1732176812677 because future has completed 2024-11-21T08:13:37,571 INFO [StoreOpener-dbfd5210cf46b2a246edeeebd244b662-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region dbfd5210cf46b2a246edeeebd244b662 2024-11-21T08:13:37,571 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=9eb0364d53b384c3e59ecc15348e6c3c, regionState=OPEN, openSeqNum=2, regionLocation=6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:37,573 INFO [StoreOpener-dbfd5210cf46b2a246edeeebd244b662-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbfd5210cf46b2a246edeeebd244b662 columnFamilyName family 2024-11-21T08:13:37,573 DEBUG [StoreOpener-dbfd5210cf46b2a246edeeebd244b662-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:37,574 INFO [StoreOpener-dbfd5210cf46b2a246edeeebd244b662-1 {}] regionserver.HStore(327): Store=dbfd5210cf46b2a246edeeebd244b662/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:37,575 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for dbfd5210cf46b2a246edeeebd244b662 2024-11-21T08:13:37,575 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9eb0364d53b384c3e59ecc15348e6c3c, server=6ed56fb6cbf7,33091,1732176812968 because future has completed 2024-11-21T08:13:37,576 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/dbfd5210cf46b2a246edeeebd244b662 2024-11-21T08:13:37,576 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/dbfd5210cf46b2a246edeeebd244b662 2024-11-21T08:13:37,577 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for dbfd5210cf46b2a246edeeebd244b662 2024-11-21T08:13:37,577 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for dbfd5210cf46b2a246edeeebd244b662 2024-11-21T08:13:37,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=24 2024-11-21T08:13:37,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure ec3a5c93577675c61d946ed41b85bdb5, server=6ed56fb6cbf7,35295,1732176812677 in 427 msec 2024-11-21T08:13:37,581 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for dbfd5210cf46b2a246edeeebd244b662 2024-11-21T08:13:37,583 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ec3a5c93577675c61d946ed41b85bdb5, ASSIGN in 660 msec 2024-11-21T08:13:37,584 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=6ed56fb6cbf7,38845,1732176812866, table=test, region=f38fd8b8defc002b7cb3de558717f27c. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T08:13:37,586 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/dbfd5210cf46b2a246edeeebd244b662/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T08:13:37,587 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=test, region=4f43187c46588bff73d01ba7eb26a6f8. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T08:13:37,587 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened dbfd5210cf46b2a246edeeebd244b662; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68129490, jitterRate=0.015208512544631958}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:37,587 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dbfd5210cf46b2a246edeeebd244b662 2024-11-21T08:13:37,587 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=test, region=99e7b3565f08f10e8c368072e9c21c09. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T08:13:37,587 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=test, region=2d5b1f002057972e76beae8f19094030. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T08:13:37,587 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for dbfd5210cf46b2a246edeeebd244b662: Running coprocessor pre-open hook at 1732176817567Writing region info on filesystem at 1732176817567Initializing all the Stores at 1732176817570 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176817570Cleaning up temporary data from old regions at 1732176817577 (+7 ms)Running coprocessor post-open hooks at 1732176817587 (+10 ms)Region opened successfully at 1732176817587 2024-11-21T08:13:37,589 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1732176816148.dbfd5210cf46b2a246edeeebd244b662., pid=52, masterSystemTime=1732176817346 2024-11-21T08:13:37,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=12 2024-11-21T08:13:37,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 9eb0364d53b384c3e59ecc15348e6c3c, server=6ed56fb6cbf7,33091,1732176812968 in 359 msec 2024-11-21T08:13:37,591 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1732176816148.dbfd5210cf46b2a246edeeebd244b662. 2024-11-21T08:13:37,592 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,kkk,1732176816148.dbfd5210cf46b2a246edeeebd244b662. 2024-11-21T08:13:37,592 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,rrr,1732176816148.f55c025b8995c15117005602a775de64. 2024-11-21T08:13:37,592 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => f55c025b8995c15117005602a775de64, NAME => 'test,rrr,1732176816148.f55c025b8995c15117005602a775de64.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-21T08:13:37,592 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f55c025b8995c15117005602a775de64 2024-11-21T08:13:37,592 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,rrr,1732176816148.f55c025b8995c15117005602a775de64.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:37,592 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for f55c025b8995c15117005602a775de64 2024-11-21T08:13:37,592 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for f55c025b8995c15117005602a775de64 2024-11-21T08:13:37,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN in 670 msec 2024-11-21T08:13:37,594 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=dbfd5210cf46b2a246edeeebd244b662, regionState=OPEN, openSeqNum=2, regionLocation=6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:37,595 INFO [StoreOpener-f55c025b8995c15117005602a775de64-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f55c025b8995c15117005602a775de64 2024-11-21T08:13:37,598 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure dbfd5210cf46b2a246edeeebd244b662, server=6ed56fb6cbf7,33091,1732176812968 because future has completed 2024-11-21T08:13:37,598 INFO [StoreOpener-f55c025b8995c15117005602a775de64-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f55c025b8995c15117005602a775de64 columnFamilyName family 2024-11-21T08:13:37,598 DEBUG [StoreOpener-f55c025b8995c15117005602a775de64-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:37,599 INFO [StoreOpener-f55c025b8995c15117005602a775de64-1 {}] regionserver.HStore(327): Store=f55c025b8995c15117005602a775de64/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:37,599 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for f55c025b8995c15117005602a775de64 2024-11-21T08:13:37,600 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/f55c025b8995c15117005602a775de64 2024-11-21T08:13:37,600 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/f55c025b8995c15117005602a775de64 2024-11-21T08:13:37,601 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for f55c025b8995c15117005602a775de64 2024-11-21T08:13:37,601 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for f55c025b8995c15117005602a775de64 2024-11-21T08:13:37,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=7 2024-11-21T08:13:37,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure f38fd8b8defc002b7cb3de558717f27c, server=6ed56fb6cbf7,38845,1732176812866 in 456 msec 2024-11-21T08:13:37,603 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for f55c025b8995c15117005602a775de64 2024-11-21T08:13:37,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f38fd8b8defc002b7cb3de558717f27c, ASSIGN in 683 msec 2024-11-21T08:13:37,606 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/f55c025b8995c15117005602a775de64/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T08:13:37,607 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened f55c025b8995c15117005602a775de64; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70861079, jitterRate=0.055912360548973083}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:37,607 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f55c025b8995c15117005602a775de64 2024-11-21T08:13:37,608 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for f55c025b8995c15117005602a775de64: Running coprocessor pre-open hook at 1732176817593Writing region info on filesystem at 1732176817593Initializing all the Stores at 1732176817594 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176817594Cleaning up temporary data from old regions at 1732176817601 (+7 ms)Running coprocessor post-open hooks at 1732176817607 (+6 ms)Region opened successfully at 1732176817607 2024-11-21T08:13:37,609 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732176816148.f55c025b8995c15117005602a775de64., pid=50, masterSystemTime=1732176817346 2024-11-21T08:13:37,612 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732176816148.f55c025b8995c15117005602a775de64. 2024-11-21T08:13:37,612 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,rrr,1732176816148.f55c025b8995c15117005602a775de64. 2024-11-21T08:13:37,612 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,,1732176816148.fd0a67dc199543d2675c547a7cb41702. 2024-11-21T08:13:37,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=15 2024-11-21T08:13:37,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure dbfd5210cf46b2a246edeeebd244b662, server=6ed56fb6cbf7,33091,1732176812968 in 376 msec 2024-11-21T08:13:37,612 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => fd0a67dc199543d2675c547a7cb41702, NAME => 'test,,1732176816148.fd0a67dc199543d2675c547a7cb41702.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-21T08:13:37,613 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fd0a67dc199543d2675c547a7cb41702 2024-11-21T08:13:37,613 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,,1732176816148.fd0a67dc199543d2675c547a7cb41702.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:37,613 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for fd0a67dc199543d2675c547a7cb41702 2024-11-21T08:13:37,613 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for fd0a67dc199543d2675c547a7cb41702 2024-11-21T08:13:37,615 INFO [StoreOpener-fd0a67dc199543d2675c547a7cb41702-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fd0a67dc199543d2675c547a7cb41702 2024-11-21T08:13:37,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN in 693 msec 2024-11-21T08:13:37,617 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=f55c025b8995c15117005602a775de64, regionState=OPEN, openSeqNum=2, regionLocation=6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:37,617 INFO [StoreOpener-fd0a67dc199543d2675c547a7cb41702-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd0a67dc199543d2675c547a7cb41702 columnFamilyName family 2024-11-21T08:13:37,617 DEBUG [StoreOpener-fd0a67dc199543d2675c547a7cb41702-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:37,618 INFO [StoreOpener-fd0a67dc199543d2675c547a7cb41702-1 {}] regionserver.HStore(327): Store=fd0a67dc199543d2675c547a7cb41702/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:37,618 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for fd0a67dc199543d2675c547a7cb41702 2024-11-21T08:13:37,620 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/fd0a67dc199543d2675c547a7cb41702 2024-11-21T08:13:37,620 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=477e65b3290a1e6e03e5eb59f7aa956d, regionState=OPEN, openSeqNum=2, regionLocation=6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:37,621 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/fd0a67dc199543d2675c547a7cb41702 2024-11-21T08:13:37,621 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure f55c025b8995c15117005602a775de64, server=6ed56fb6cbf7,33091,1732176812968 because future has completed 2024-11-21T08:13:37,622 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for fd0a67dc199543d2675c547a7cb41702 2024-11-21T08:13:37,622 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for fd0a67dc199543d2675c547a7cb41702 2024-11-21T08:13:37,624 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 477e65b3290a1e6e03e5eb59f7aa956d, server=6ed56fb6cbf7,33091,1732176812968 because future has completed 2024-11-21T08:13:37,625 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for fd0a67dc199543d2675c547a7cb41702 2024-11-21T08:13:37,628 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/fd0a67dc199543d2675c547a7cb41702/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T08:13:37,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=22 2024-11-21T08:13:37,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure f55c025b8995c15117005602a775de64, server=6ed56fb6cbf7,33091,1732176812968 in 405 msec 2024-11-21T08:13:37,629 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened fd0a67dc199543d2675c547a7cb41702; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66492269, jitterRate=-0.009187981486320496}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:37,629 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fd0a67dc199543d2675c547a7cb41702 2024-11-21T08:13:37,629 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for fd0a67dc199543d2675c547a7cb41702: Running coprocessor pre-open hook at 1732176817613Writing region info on filesystem at 1732176817613Initializing all the Stores at 1732176817614 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176817614Cleaning up temporary data from old regions at 1732176817622 (+8 ms)Running coprocessor post-open hooks at 1732176817629 (+7 ms)Region opened successfully at 1732176817629 2024-11-21T08:13:37,630 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1732176816148.fd0a67dc199543d2675c547a7cb41702., pid=55, masterSystemTime=1732176817346 2024-11-21T08:13:37,632 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN in 709 msec 2024-11-21T08:13:37,633 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1732176816148.fd0a67dc199543d2675c547a7cb41702. 2024-11-21T08:13:37,633 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,,1732176816148.fd0a67dc199543d2675c547a7cb41702. 2024-11-21T08:13:37,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=19 2024-11-21T08:13:37,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 477e65b3290a1e6e03e5eb59f7aa956d, server=6ed56fb6cbf7,33091,1732176812968 in 398 msec 2024-11-21T08:13:37,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN in 716 msec 2024-11-21T08:13:37,639 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fd0a67dc199543d2675c547a7cb41702, regionState=OPEN, openSeqNum=2, regionLocation=6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:37,642 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd0a67dc199543d2675c547a7cb41702, server=6ed56fb6cbf7,33091,1732176812968 because future has completed 2024-11-21T08:13:37,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=21 2024-11-21T08:13:37,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 2d5b1f002057972e76beae8f19094030, server=6ed56fb6cbf7,35295,1732176812677 in 484 msec 2024-11-21T08:13:37,648 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2d5b1f002057972e76beae8f19094030, ASSIGN in 726 msec 2024-11-21T08:13:37,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=5 2024-11-21T08:13:37,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure fd0a67dc199543d2675c547a7cb41702, server=6ed56fb6cbf7,33091,1732176812968 in 400 msec 2024-11-21T08:13:37,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN in 731 msec 2024-11-21T08:13:37,655 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=021eaa39c912c55ccf9b758a5c92d1f8, regionState=OPEN, openSeqNum=2, regionLocation=6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:37,658 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 021eaa39c912c55ccf9b758a5c92d1f8, server=6ed56fb6cbf7,33091,1732176812968 because future has completed 2024-11-21T08:13:37,659 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=5fb158c8201cfbc5babdbfc33ee45ab1, regionState=OPEN, openSeqNum=2, regionLocation=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:37,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5fb158c8201cfbc5babdbfc33ee45ab1, server=6ed56fb6cbf7,35295,1732176812677 because future has completed 2024-11-21T08:13:37,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=28 2024-11-21T08:13:37,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 021eaa39c912c55ccf9b758a5c92d1f8, server=6ed56fb6cbf7,33091,1732176812968 in 462 msec 2024-11-21T08:13:37,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN in 745 msec 2024-11-21T08:13:37,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=14 2024-11-21T08:13:37,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 5fb158c8201cfbc5babdbfc33ee45ab1, server=6ed56fb6cbf7,35295,1732176812677 in 492 msec 2024-11-21T08:13:37,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5fb158c8201cfbc5babdbfc33ee45ab1, ASSIGN in 752 msec 2024-11-21T08:13:37,675 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=3773623806176f3d7f8efb3efe1b7937, regionState=OPEN, openSeqNum=2, regionLocation=6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:37,678 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3773623806176f3d7f8efb3efe1b7937, server=6ed56fb6cbf7,33091,1732176812968 because future has completed 2024-11-21T08:13:37,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=8 2024-11-21T08:13:37,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 4f43187c46588bff73d01ba7eb26a6f8, server=6ed56fb6cbf7,35295,1732176812677 in 471 msec 2024-11-21T08:13:37,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4f43187c46588bff73d01ba7eb26a6f8, ASSIGN in 762 msec 2024-11-21T08:13:37,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=6 2024-11-21T08:13:37,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 3773623806176f3d7f8efb3efe1b7937, server=6ed56fb6cbf7,33091,1732176812968 in 426 msec 2024-11-21T08:13:37,691 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=test, region=99e7b3565f08f10e8c368072e9c21c09. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T08:13:37,693 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN in 771 msec 2024-11-21T08:13:37,696 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=18 2024-11-21T08:13:37,696 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 99e7b3565f08f10e8c368072e9c21c09, server=6ed56fb6cbf7,35295,1732176812677 in 540 msec 2024-11-21T08:13:37,698 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=99e7b3565f08f10e8c368072e9c21c09, ASSIGN in 777 msec 2024-11-21T08:13:37,699 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=d72691fc99dffac743de3d6bcb0f84ed, regionState=OPEN, openSeqNum=2, regionLocation=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:37,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure d72691fc99dffac743de3d6bcb0f84ed, server=6ed56fb6cbf7,38845,1732176812866 because future has completed 2024-11-21T08:13:37,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=17 2024-11-21T08:13:37,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure d72691fc99dffac743de3d6bcb0f84ed, server=6ed56fb6cbf7,38845,1732176812866 in 601 msec 2024-11-21T08:13:37,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=4 2024-11-21T08:13:37,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d72691fc99dffac743de3d6bcb0f84ed, ASSIGN in 785 msec 2024-11-21T08:13:37,709 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T08:13:37,710 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732176817709"}]},"ts":"1732176817709"} 2024-11-21T08:13:37,713 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-21T08:13:37,714 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T08:13:37,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.5560 sec 2024-11-21T08:13:38,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T08:13:38,339 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-21T08:13:38,339 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-21T08:13:38,340 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T08:13:38,364 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-21T08:13:38,364 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T08:13:38,365 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-21T08:13:38,366 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T08:13:38,372 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-21T08:13:38,401 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-21T08:13:38,422 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T08:13:38,425 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T08:13:38,427 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T08:13:38,436 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T08:13:38,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fd0a67dc199543d2675c547a7cb41702 changed from -1.0 to 0.0, refreshing cache 2024-11-21T08:13:38,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3773623806176f3d7f8efb3efe1b7937 changed from -1.0 to 0.0, refreshing cache 2024-11-21T08:13:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region cd29f056dad8c6c37d0979434891505d changed from -1.0 to 0.0, refreshing cache 2024-11-21T08:13:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9eb0364d53b384c3e59ecc15348e6c3c changed from -1.0 to 0.0, refreshing cache 2024-11-21T08:13:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region dbfd5210cf46b2a246edeeebd244b662 changed from -1.0 to 0.0, refreshing cache 2024-11-21T08:13:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 477e65b3290a1e6e03e5eb59f7aa956d changed from -1.0 to 0.0, refreshing cache 2024-11-21T08:13:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f55c025b8995c15117005602a775de64 changed from -1.0 to 0.0, refreshing cache 2024-11-21T08:13:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7ef5fca53c8ff0d12fc112ab68731575 changed from -1.0 to 0.0, refreshing cache 2024-11-21T08:13:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 021eaa39c912c55ccf9b758a5c92d1f8 changed from -1.0 to 0.0, refreshing cache 2024-11-21T08:13:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4f43187c46588bff73d01ba7eb26a6f8 changed from -1.0 to 0.0, refreshing cache 2024-11-21T08:13:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ee7ef14c3df8915e550cfda528563489 changed from -1.0 to 0.0, refreshing cache 2024-11-21T08:13:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5fb158c8201cfbc5babdbfc33ee45ab1 changed from -1.0 to 0.0, refreshing cache 2024-11-21T08:13:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 99e7b3565f08f10e8c368072e9c21c09 changed from -1.0 to 0.0, refreshing cache 2024-11-21T08:13:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2d5b1f002057972e76beae8f19094030 changed from -1.0 to 0.0, refreshing cache 2024-11-21T08:13:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ec3a5c93577675c61d946ed41b85bdb5 changed from -1.0 to 0.0, refreshing cache 2024-11-21T08:13:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ebf8202a2e0e04a2c5c7f5f0e491d40e changed from -1.0 to 0.0, refreshing cache 2024-11-21T08:13:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3589ece79012bcc3080c57b249902675 changed from -1.0 to 0.0, refreshing cache 2024-11-21T08:13:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f38fd8b8defc002b7cb3de558717f27c changed from -1.0 to 0.0, refreshing cache 2024-11-21T08:13:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 829529fdccf9853d96113fa67713ffbc changed from -1.0 to 0.0, refreshing cache 2024-11-21T08:13:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fd97d8f813bdfcba3626cc4950c7c1e1 changed from -1.0 to 0.0, refreshing cache 2024-11-21T08:13:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 94c604fd0d435e28ac643b49d6cbee14 changed from -1.0 to 0.0, refreshing cache 2024-11-21T08:13:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d72691fc99dffac743de3d6bcb0f84ed changed from -1.0 to 0.0, refreshing cache 2024-11-21T08:13:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5017ae3acb74fe8c7d0b9c6d9a2cc402 changed from -1.0 to 0.0, refreshing cache 2024-11-21T08:13:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4ed3111e35326fc2c5455ae2475ab1b5 changed from -1.0 to 0.0, refreshing cache 2024-11-21T08:13:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8dfe569750c256f741c6d3e5ed072503 changed from -1.0 to 0.0, refreshing cache 2024-11-21T08:13:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region af43494f9c57dd49b6d504756610f96c changed from -1.0 to 0.0, refreshing cache 2024-11-21T08:13:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-21T08:13:38,513 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-21T08:13:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-21T08:13:38,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.BalancerClusterState(204): Hosts are {6ed56fb6cbf7=0} racks are {/default-rack=0} 2024-11-21T08:13:38,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.BalancerClusterState(303): server 0 has 10 regions 2024-11-21T08:13:38,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.BalancerClusterState(303): server 1 has 8 regions 2024-11-21T08:13:38,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-21T08:13:38,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T08:13:38,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T08:13:38,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-21T08:13:38,515 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T08:13:38,515 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T08:13:38,515 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-21T08:13:38,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-21T08:13:38,521 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.StochasticLoadBalancer(403): Cluster wide - Calculating plan. may take up to 30000ms to complete. 2024-11-21T08:13:38,523 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.StochasticLoadBalancer(515): Start StochasticLoadBalancer.balancer, initial weighted average imbalance=0.06931633435746164, functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.06415002990995843, need balance); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); computedMaxSteps=64800 2024-11-21T08:13:39,140 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] balancer.StochasticLoadBalancer(562): Finished computing new moving plan. Computation took 624 ms to try 64800 different iterations. Found a solution that moves 1 regions; Going from a computed imbalance of 0.06931633435746164 to a new imbalance of 0.01690158032826896. funtionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.037037037037037035, need balance); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-21T08:13:39,143 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] master.HMaster(2167): Balancer plans size is 1, the balance interval is 300000 ms, and the max number regions in transition is 27 2024-11-21T08:13:39,144 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] master.HMaster(2172): balance hri=7ef5fca53c8ff0d12fc112ab68731575, source=6ed56fb6cbf7,33091,1732176812968, destination=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:39,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, REOPEN/MOVE 2024-11-21T08:13:39,146 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, REOPEN/MOVE 2024-11-21T08:13:39,149 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=7ef5fca53c8ff0d12fc112ab68731575, regionState=CLOSING, regionLocation=6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:39,153 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, REOPEN/MOVE because future has completed 2024-11-21T08:13:39,153 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T08:13:39,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7ef5fca53c8ff0d12fc112ab68731575, server=6ed56fb6cbf7,33091,1732176812968}] 2024-11-21T08:13:39,312 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(122): Close 7ef5fca53c8ff0d12fc112ab68731575 2024-11-21T08:13:39,312 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T08:13:39,313 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1722): Closing 7ef5fca53c8ff0d12fc112ab68731575, disabling compactions & flushes 2024-11-21T08:13:39,313 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1755): Closing region test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575. 2024-11-21T08:13:39,313 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575. 2024-11-21T08:13:39,313 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575. after waiting 0 ms 2024-11-21T08:13:39,313 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575. 2024-11-21T08:13:39,318 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/7ef5fca53c8ff0d12fc112ab68731575/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T08:13:39,320 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1973): Closed test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575. 2024-11-21T08:13:39,320 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1676): Region close journal for 7ef5fca53c8ff0d12fc112ab68731575: Waiting for close lock at 1732176819313Running coprocessor pre-close hooks at 1732176819313Disabling compacts and flushes for region at 1732176819313Disabling writes for close at 1732176819313Writing region close event to WAL at 1732176819313Running coprocessor post-close hooks at 1732176819319 (+6 ms)Closed at 1732176819320 (+1 ms) 2024-11-21T08:13:39,321 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionServer(3302): Adding 7ef5fca53c8ff0d12fc112ab68731575 move to 6ed56fb6cbf7,35295,1732176812677 record at close sequenceid=2 2024-11-21T08:13:39,326 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(157): Closed 7ef5fca53c8ff0d12fc112ab68731575 2024-11-21T08:13:39,326 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=7ef5fca53c8ff0d12fc112ab68731575, regionState=CLOSED 2024-11-21T08:13:39,329 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7ef5fca53c8ff0d12fc112ab68731575, server=6ed56fb6cbf7,33091,1732176812968 because future has completed 2024-11-21T08:13:39,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-21T08:13:39,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; CloseRegionProcedure 7ef5fca53c8ff0d12fc112ab68731575, server=6ed56fb6cbf7,33091,1732176812968 in 177 msec 2024-11-21T08:13:39,333 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, REOPEN/MOVE; state=CLOSED, location=6ed56fb6cbf7,35295,1732176812677; forceNewPlan=false, retain=false 2024-11-21T08:13:39,484 INFO [6ed56fb6cbf7:32901 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-21T08:13:39,485 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=7ef5fca53c8ff0d12fc112ab68731575, regionState=OPENING, regionLocation=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:39,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, REOPEN/MOVE because future has completed 2024-11-21T08:13:39,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ef5fca53c8ff0d12fc112ab68731575, server=6ed56fb6cbf7,35295,1732176812677}] 2024-11-21T08:13:39,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741828_1004 (size=1189) 2024-11-21T08:13:39,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741840_1016 (size=45) 2024-11-21T08:13:39,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741829_1005 (size=34) 2024-11-21T08:13:39,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741832_1008 (size=32) 2024-11-21T08:13:39,648 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575. 2024-11-21T08:13:39,648 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 7ef5fca53c8ff0d12fc112ab68731575, NAME => 'test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-21T08:13:39,649 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7ef5fca53c8ff0d12fc112ab68731575 2024-11-21T08:13:39,649 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:39,649 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for 7ef5fca53c8ff0d12fc112ab68731575 2024-11-21T08:13:39,649 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for 7ef5fca53c8ff0d12fc112ab68731575 2024-11-21T08:13:39,651 INFO [StoreOpener-7ef5fca53c8ff0d12fc112ab68731575-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7ef5fca53c8ff0d12fc112ab68731575 2024-11-21T08:13:39,652 INFO [StoreOpener-7ef5fca53c8ff0d12fc112ab68731575-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7ef5fca53c8ff0d12fc112ab68731575 columnFamilyName family 2024-11-21T08:13:39,653 DEBUG [StoreOpener-7ef5fca53c8ff0d12fc112ab68731575-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:39,653 INFO [StoreOpener-7ef5fca53c8ff0d12fc112ab68731575-1 {}] regionserver.HStore(327): Store=7ef5fca53c8ff0d12fc112ab68731575/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:39,654 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for 7ef5fca53c8ff0d12fc112ab68731575 2024-11-21T08:13:39,655 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/7ef5fca53c8ff0d12fc112ab68731575 2024-11-21T08:13:39,656 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/7ef5fca53c8ff0d12fc112ab68731575 2024-11-21T08:13:39,657 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for 7ef5fca53c8ff0d12fc112ab68731575 2024-11-21T08:13:39,657 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 7ef5fca53c8ff0d12fc112ab68731575 2024-11-21T08:13:39,659 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for 7ef5fca53c8ff0d12fc112ab68731575 2024-11-21T08:13:39,660 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened 7ef5fca53c8ff0d12fc112ab68731575; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74297201, jitterRate=0.10711456835269928}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:39,660 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7ef5fca53c8ff0d12fc112ab68731575 2024-11-21T08:13:39,660 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for 7ef5fca53c8ff0d12fc112ab68731575: Running coprocessor pre-open hook at 1732176819649Writing region info on filesystem at 1732176819649Initializing all the Stores at 1732176819651 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176819651Cleaning up temporary data from old regions at 1732176819657 (+6 ms)Running coprocessor post-open hooks at 1732176819660 (+3 ms)Region opened successfully at 1732176819660 2024-11-21T08:13:39,661 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575., pid=59, masterSystemTime=1732176819644 2024-11-21T08:13:39,664 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575. 2024-11-21T08:13:39,664 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575. 2024-11-21T08:13:39,665 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=7ef5fca53c8ff0d12fc112ab68731575, regionState=OPEN, openSeqNum=5, regionLocation=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:39,668 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ef5fca53c8ff0d12fc112ab68731575, server=6ed56fb6cbf7,35295,1732176812677 because future has completed 2024-11-21T08:13:39,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=57 2024-11-21T08:13:39,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=57, state=SUCCESS, hasLock=false; OpenRegionProcedure 7ef5fca53c8ff0d12fc112ab68731575, server=6ed56fb6cbf7,35295,1732176812677 in 179 msec 2024-11-21T08:13:39,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, REOPEN/MOVE in 528 msec 2024-11-21T08:13:39,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-21T08:13:39,750 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T08:13:39,757 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.3 set balanceSwitch=false 2024-11-21T08:13:39,771 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=383, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=671, ProcessCount=11, AvailableMemoryMB=5440 2024-11-21T08:13:39,774 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-21T08:13:39,774 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-21T08:13:39,775 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:39,776 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 6ed56fb6cbf7,33091,1732176812968: Simulated kill ***** 2024-11-21T08:13:39,776 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-21T08:13:39,778 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-21T08:13:39,782 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-21T08:13:39,782 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-21T08:13:39,782 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-21T08:13:39,783 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "Verbose": false, "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 336059944 }, "NonHeapMemoryUsage": { "committed": 154206208, "init": 7667712, "max": -1, "used": 151841976 }, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "6ed56fb6cbf7" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "6ed56fb6cbf7" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "6ed56fb6cbf7" } ] } 2024-11-21T08:13:39,786 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32901 {}] master.MasterRpcServices(700): 6ed56fb6cbf7,33091,1732176812968 reported a fatal error: ***** ABORTING region server 6ed56fb6cbf7,33091,1732176812968: Simulated kill ***** 2024-11-21T08:13:39,789 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6ed56fb6cbf7,33091,1732176812968' ***** 2024-11-21T08:13:39,789 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-21T08:13:39,789 INFO [RS:2;6ed56fb6cbf7:33091 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T08:13:39,790 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T08:13:39,790 INFO [RS:2;6ed56fb6cbf7:33091 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-21T08:13:39,790 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T08:13:39,790 INFO [RS:2;6ed56fb6cbf7:33091 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-21T08:13:39,790 INFO [RS:2;6ed56fb6cbf7:33091 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T08:13:39,790 INFO [RS:2;6ed56fb6cbf7:33091 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;6ed56fb6cbf7:33091. 2024-11-21T08:13:39,791 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T08:13:39,798 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T08:13:39,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741836_1012 (size=33749) 2024-11-21T08:13:39,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741836_1012 (size=33749) 2024-11-21T08:13:39,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741836_1012 (size=33749) 2024-11-21T08:13:39,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741833_1009 (size=2769) 2024-11-21T08:13:39,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741833_1009 (size=2769) 2024-11-21T08:13:39,820 DEBUG [RS:2;6ed56fb6cbf7:33091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T08:13:39,821 INFO [RS:2;6ed56fb6cbf7:33091 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T08:13:39,821 INFO [RS:2;6ed56fb6cbf7:33091 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:33091 2024-11-21T08:13:39,840 INFO [regionserver/6ed56fb6cbf7:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-21T08:13:39,840 INFO [regionserver/6ed56fb6cbf7:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-21T08:13:39,852 INFO [regionserver/6ed56fb6cbf7:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T08:13:39,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T08:13:39,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x1015c748eb50003, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6ed56fb6cbf7,33091,1732176812968 2024-11-21T08:13:39,883 INFO [RS:2;6ed56fb6cbf7:33091 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T08:13:39,884 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6ed56fb6cbf7,33091,1732176812968] 2024-11-21T08:13:39,938 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6ed56fb6cbf7,33091,1732176812968 already deleted, retry=false 2024-11-21T08:13:39,939 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 6ed56fb6cbf7,33091,1732176812968 on 6ed56fb6cbf7,32901,1732176811582 2024-11-21T08:13:39,945 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true 2024-11-21T08:13:39,950 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=60 for 6ed56fb6cbf7,33091,1732176812968 (carryingMeta=true) 6ed56fb6cbf7,33091,1732176812968/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@25659421[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-21T08:13:39,951 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true 2024-11-21T08:13:39,953 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true, isMeta: true 2024-11-21T08:13:39,957 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968-splitting 2024-11-21T08:13:39,958 INFO [PEWorker-1 {}] master.SplitWALManager(105): 6ed56fb6cbf7,33091,1732176812968 WAL count=1, meta=true 2024-11-21T08:13:39,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta}] 2024-11-21T08:13:39,969 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:39,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866}] 2024-11-21T08:13:39,993 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T08:13:39,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x1015c748eb50003, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T08:13:39,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33091-0x1015c748eb50003, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T08:13:39,999 INFO [RS:2;6ed56fb6cbf7:33091 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T08:13:40,000 INFO [RS:2;6ed56fb6cbf7:33091 {}] regionserver.HRegionServer(1031): Exiting; stopping=6ed56fb6cbf7,33091,1732176812968; zookeeper connection closed. 2024-11-21T08:13:40,000 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@55004014 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@55004014 2024-11-21T08:13:40,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38845 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=62 2024-11-21T08:13:40,163 INFO [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(299): Splitting hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968-splitting/6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, size=33.0 K (33749bytes) 2024-11-21T08:13:40,164 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-21T08:13:40,164 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-21T08:13:40,164 INFO [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968-splitting/6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta 2024-11-21T08:13:40,169 INFO [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968-splitting/6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta after 3ms 2024-11-21T08:13:40,176 INFO [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(310): Open hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968-splitting/6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta took 14ms 2024-11-21T08:13:40,185 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-21T08:13:40,191 INFO [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta.temp 2024-11-21T08:13:40,194 INFO [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta.temp 2024-11-21T08:13:40,196 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968-splitting/6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta so closing down 2024-11-21T08:13:40,196 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-21T08:13:40,199 INFO [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(145): 3 split writer threads finished 2024-11-21T08:13:40,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741864_1040 (size=33508) 2024-11-21T08:13:40,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741864_1040 (size=33508) 2024-11-21T08:13:40,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741864_1040 (size=33508) 2024-11-21T08:13:40,211 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta.temp (wrote 61 edits, skipped 0 edits in 3 ms) 2024-11-21T08:13:40,215 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta.temp to hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-21T08:13:40,219 INFO [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(425): Processed 62 edits across 1 Regions in 38 ms; skipped=1; WAL=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968-splitting/6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false 2024-11-21T08:13:40,220 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(428): Completed split of hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968-splitting/6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, journal: Splitting hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968-splitting/6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, size=33.0 K (33749bytes) at 1732176820163Creating recovered edits writer path=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta.temp at 1732176820194 (+31 ms)Finishing writing output for hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968-splitting/6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta so closing down at 1732176820196 (+2 ms)3 split writer threads finished at 1732176820199 (+3 ms)Closed recovered edits writer path=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta.temp (wrote 61 edits, skipped 0 edits in 3 ms) at 1732176820212 (+13 ms)Rename recovered edits hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta.temp to hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 at 1732176820215 (+3 ms)Processed 62 edits across 1 Regions in 38 ms; skipped=1; WAL=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968-splitting/6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false at 1732176820219 (+4 ms) 2024-11-21T08:13:40,220 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968-splitting/6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta 2024-11-21T08:13:40,222 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-11-21T08:13:40,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-11-21T08:13:40,232 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968-splitting/6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta to hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/oldWALs 2024-11-21T08:13:40,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-21T08:13:40,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 in 261 msec 2024-11-21T08:13:40,236 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:40,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-21T08:13:40,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 in 277 msec 2024-11-21T08:13:40,242 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968-splitting dir is empty, no logs to split. 2024-11-21T08:13:40,242 INFO [PEWorker-1 {}] master.SplitWALManager(105): 6ed56fb6cbf7,33091,1732176812968 WAL count=0, meta=true 2024-11-21T08:13:40,242 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 6ed56fb6cbf7,33091,1732176812968 WAL splitting is done? wals=0, meta=true 2024-11-21T08:13:40,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T08:13:40,246 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T08:13:40,247 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T08:13:40,398 DEBUG [6ed56fb6cbf7:32901 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-21T08:13:40,398 DEBUG [6ed56fb6cbf7:32901 {}] balancer.BalancerClusterState(204): Hosts are {6ed56fb6cbf7=0} racks are {/default-rack=0} 2024-11-21T08:13:40,398 DEBUG [6ed56fb6cbf7:32901 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T08:13:40,398 DEBUG [6ed56fb6cbf7:32901 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T08:13:40,398 DEBUG [6ed56fb6cbf7:32901 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T08:13:40,398 DEBUG [6ed56fb6cbf7:32901 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T08:13:40,398 INFO [6ed56fb6cbf7:32901 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T08:13:40,398 INFO [6ed56fb6cbf7:32901 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T08:13:40,398 DEBUG [6ed56fb6cbf7:32901 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T08:13:40,399 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:40,400 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ed56fb6cbf7,35295,1732176812677, state=OPENING 2024-11-21T08:13:40,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:40,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:40,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:40,424 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T08:13:40,425 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:40,425 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:40,425 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:40,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677}] 2024-11-21T08:13:40,586 INFO [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T08:13:40,586 INFO [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T08:13:40,587 INFO [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T08:13:40,590 INFO [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6ed56fb6cbf7%2C35295%2C1732176812677.meta, suffix=.meta, logDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,35295,1732176812677, archiveDir=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/oldWALs, maxLogs=32 2024-11-21T08:13:40,611 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,35295,1732176812677/6ed56fb6cbf7%2C35295%2C1732176812677.meta.1732176820592.meta, exclude list is [], retry=0 2024-11-21T08:13:40,614 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-9db7f2dc-3c49-4f47-8420-8e992a2830c6,DISK] 2024-11-21T08:13:40,614 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33295,DS-7db7df2d-be4b-495c-9b4b-5548750637bb,DISK] 2024-11-21T08:13:40,615 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34217,DS-f098b1c9-3781-437c-aa42-0db9fe2ba16f,DISK] 2024-11-21T08:13:40,617 INFO [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,35295,1732176812677/6ed56fb6cbf7%2C35295%2C1732176812677.meta.1732176820592.meta 2024-11-21T08:13:40,617 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46533:46533),(127.0.0.1/127.0.0.1:45803:45803),(127.0.0.1/127.0.0.1:37401:37401)] 2024-11-21T08:13:40,617 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T08:13:40,618 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T08:13:40,618 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T08:13:40,618 INFO [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T08:13:40,618 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T08:13:40,618 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:40,618 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T08:13:40,618 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T08:13:40,620 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T08:13:40,621 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T08:13:40,621 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:40,622 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T08:13:40,622 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T08:13:40,623 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T08:13:40,623 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:40,624 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T08:13:40,624 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T08:13:40,625 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T08:13:40,625 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:40,625 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T08:13:40,625 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T08:13:40,626 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T08:13:40,626 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:40,627 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T08:13:40,627 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T08:13:40,628 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740 2024-11-21T08:13:40,630 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740 2024-11-21T08:13:40,631 INFO [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-21T08:13:40,644 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5793): Applied 277, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=64, path=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-21T08:13:40,647 INFO [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=29.41 KB heapSize=50.67 KB 2024-11-21T08:13:40,721 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/.tmp/info/3735ebac270441429dd9557c51fc07c4 is 138, key is test,bbb,1732176816148.3773623806176f3d7f8efb3efe1b7937./info:regioninfo/1732176817675/Put/seqid=0 2024-11-21T08:13:40,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741866_1042 (size=36371) 2024-11-21T08:13:40,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741866_1042 (size=36371) 2024-11-21T08:13:40,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741866_1042 (size=36371) 2024-11-21T08:13:40,735 INFO [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.26 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/.tmp/info/3735ebac270441429dd9557c51fc07c4 2024-11-21T08:13:40,795 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/.tmp/ns/f07d1339e99d4467afd1ada0de8b905e is 43, key is default/ns:d/1732176815741/Put/seqid=0 2024-11-21T08:13:40,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741867_1043 (size=5153) 2024-11-21T08:13:40,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741867_1043 (size=5153) 2024-11-21T08:13:40,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741867_1043 (size=5153) 2024-11-21T08:13:40,803 INFO [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/.tmp/ns/f07d1339e99d4467afd1ada0de8b905e 2024-11-21T08:13:40,826 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/.tmp/table/318b33340b984fadb16e03cc38ff7397 is 40, key is test/table:state/1732176817709/Put/seqid=0 2024-11-21T08:13:40,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741868_1044 (size=5165) 2024-11-21T08:13:40,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741868_1044 (size=5165) 2024-11-21T08:13:40,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741868_1044 (size=5165) 2024-11-21T08:13:40,836 INFO [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/.tmp/table/318b33340b984fadb16e03cc38ff7397 2024-11-21T08:13:40,845 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/.tmp/info/3735ebac270441429dd9557c51fc07c4 as hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/info/3735ebac270441429dd9557c51fc07c4 2024-11-21T08:13:40,854 INFO [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/info/3735ebac270441429dd9557c51fc07c4, entries=273, sequenceid=64, filesize=35.5 K 2024-11-21T08:13:40,857 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/.tmp/ns/f07d1339e99d4467afd1ada0de8b905e as hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/ns/f07d1339e99d4467afd1ada0de8b905e 2024-11-21T08:13:40,868 INFO [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/ns/f07d1339e99d4467afd1ada0de8b905e, entries=2, sequenceid=64, filesize=5.0 K 2024-11-21T08:13:40,870 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/.tmp/table/318b33340b984fadb16e03cc38ff7397 as hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/table/318b33340b984fadb16e03cc38ff7397 2024-11-21T08:13:40,881 INFO [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/table/318b33340b984fadb16e03cc38ff7397, entries=2, sequenceid=64, filesize=5.0 K 2024-11-21T08:13:40,887 INFO [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(3140): Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 236ms, sequenceid=64, compaction requested=false; wal=null 2024-11-21T08:13:40,888 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-21T08:13:40,892 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-21T08:13:40,894 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T08:13:40,895 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T08:13:40,896 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T08:13:40,905 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T08:13:40,908 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/recovered.edits/64.seqid, newMaxSeqId=64, maxSeqId=1 2024-11-21T08:13:40,910 INFO [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=65; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61006112, jitterRate=-0.09093809127807617}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T08:13:40,910 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T08:13:40,910 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732176820618Writing region info on filesystem at 1732176820618Initializing all the Stores at 1732176820620 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732176820620Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732176820620Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176820620Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732176820620Obtaining lock to block concurrent updates at 1732176820647 (+27 ms)Preparing flush snapshotting stores in 1588230740 at 1732176820647Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=30111, getHeapSize=51824, getOffHeapSize=0, getCellsCount=277 at 1732176820650 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732176820650Flushing 1588230740/info: creating writer at 1732176820651 (+1 ms)Flushing 1588230740/info: appending metadata at 1732176820708 (+57 ms)Flushing 1588230740/info: closing flushed file at 1732176820711 (+3 ms)Flushing 1588230740/ns: creating writer at 1732176820779 (+68 ms)Flushing 1588230740/ns: appending metadata at 1732176820794 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732176820794Flushing 1588230740/table: creating writer at 1732176820811 (+17 ms)Flushing 1588230740/table: appending metadata at 1732176820826 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732176820826Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d7496bb: reopening flushed file at 1732176820844 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61a7eb58: reopening flushed file at 1732176820856 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60c56269: reopening flushed file at 1732176820869 (+13 ms)Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 236ms, sequenceid=64, compaction requested=false; wal=null at 1732176820887 (+18 ms)Cleaning up temporary data from old regions at 1732176820895 (+8 ms)Running coprocessor post-open hooks at 1732176820910 (+15 ms)Region opened successfully at 1732176820910 2024-11-21T08:13:40,912 INFO [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=64, masterSystemTime=1732176820579 2024-11-21T08:13:40,915 DEBUG [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T08:13:40,915 INFO [RS_OPEN_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T08:13:40,916 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=65, regionLocation=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:40,918 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ed56fb6cbf7,35295,1732176812677, state=OPEN 2024-11-21T08:13:40,971 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T08:13:40,975 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T08:13:40,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:40,993 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=64, ppid=63, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:40,994 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:40,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:40,994 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:40,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:40,995 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:41,014 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-21T08:13:41,014 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 in 568 msec 2024-11-21T08:13:41,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=60 2024-11-21T08:13:41,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 770 msec 2024-11-21T08:13:41,021 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 6ed56fb6cbf7,33091,1732176812968 had 9 regions 2024-11-21T08:13:41,023 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true, isMeta: false 2024-11-21T08:13:41,026 INFO [PEWorker-1 {}] master.SplitWALManager(105): 6ed56fb6cbf7,33091,1732176812968 WAL count=1, meta=false 2024-11-21T08:13:41,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010}] 2024-11-21T08:13:41,029 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677}] 2024-11-21T08:13:41,040 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-21T08:13:41,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35295 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=66 2024-11-21T08:13:41,210 INFO [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(299): Splitting hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968-splitting/6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, size=2.7 K (2769bytes) 2024-11-21T08:13:41,210 INFO [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968-splitting/6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010 2024-11-21T08:13:41,211 INFO [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968-splitting/6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010 after 1ms 2024-11-21T08:13:41,215 INFO [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(310): Open hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968-splitting/6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010 took 6ms 2024-11-21T08:13:41,219 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 7ef5fca53c8ff0d12fc112ab68731575: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T08:13:41,222 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for cd29f056dad8c6c37d0979434891505d: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T08:13:41,228 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 3773623806176f3d7f8efb3efe1b7937: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T08:13:41,237 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 021eaa39c912c55ccf9b758a5c92d1f8: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T08:13:41,240 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 477e65b3290a1e6e03e5eb59f7aa956d: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T08:13:41,244 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 9eb0364d53b384c3e59ecc15348e6c3c: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T08:13:41,246 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for dbfd5210cf46b2a246edeeebd244b662: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T08:13:41,248 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for f55c025b8995c15117005602a775de64: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T08:13:41,250 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for fd0a67dc199543d2675c547a7cb41702: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T08:13:41,250 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968-splitting/6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010 so closing down 2024-11-21T08:13:41,250 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-21T08:13:41,250 INFO [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(145): 3 split writer threads finished 2024-11-21T08:13:41,250 INFO [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(425): Processed 10 edits across 0 Regions in 34 ms; skipped=10; WAL=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968-splitting/6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, size=2.7 K, length=2769, corrupted=false, cancelled=false 2024-11-21T08:13:41,251 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(428): Completed split of hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968-splitting/6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, journal: Splitting hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968-splitting/6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, size=2.7 K (2769bytes) at 1732176821210Finishing writing output for hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968-splitting/6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010 so closing down at 1732176821250 (+40 ms)3 split writer threads finished at 1732176821250Processed 10 edits across 0 Regions in 34 ms; skipped=10; WAL=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968-splitting/6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, size=2.7 K, length=2769, corrupted=false, cancelled=false at 1732176821250 2024-11-21T08:13:41,251 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968-splitting/6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010 2024-11-21T08:13:41,251 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ed56fb6cbf7:0-0 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-11-21T08:13:41,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32901 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-11-21T08:13:41,257 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968-splitting/6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010 to hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/oldWALs 2024-11-21T08:13:41,264 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-21T08:13:41,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 in 226 msec 2024-11-21T08:13:41,266 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,268 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=60 2024-11-21T08:13:41,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 in 240 msec 2024-11-21T08:13:41,271 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/WALs/6ed56fb6cbf7,33091,1732176812968-splitting dir is empty, no logs to split. 2024-11-21T08:13:41,271 INFO [PEWorker-1 {}] master.SplitWALManager(105): 6ed56fb6cbf7,33091,1732176812968 WAL count=0, meta=false 2024-11-21T08:13:41,271 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 6ed56fb6cbf7,33091,1732176812968 WAL splitting is done? wals=0, meta=false 2024-11-21T08:13:41,273 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN}, {pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN}, {pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN}, {pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN}, {pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN}, {pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN}, {pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN}, {pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN}] 2024-11-21T08:13:41,275 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN 2024-11-21T08:13:41,277 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T08:13:41,278 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN 2024-11-21T08:13:41,279 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T08:13:41,282 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN 2024-11-21T08:13:41,284 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T08:13:41,286 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN 2024-11-21T08:13:41,288 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T08:13:41,289 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN 2024-11-21T08:13:41,291 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T08:13:41,292 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:41,293 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T08:13:41,294 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:41,294 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T08:13:41,296 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:41,297 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T08:13:41,427 DEBUG [6ed56fb6cbf7:32901 {}] balancer.BalancerClusterState(204): Hosts are {6ed56fb6cbf7=0} racks are {/default-rack=0} 2024-11-21T08:13:41,427 DEBUG [6ed56fb6cbf7:32901 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T08:13:41,428 DEBUG [6ed56fb6cbf7:32901 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T08:13:41,428 DEBUG [6ed56fb6cbf7:32901 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T08:13:41,428 DEBUG [6ed56fb6cbf7:32901 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T08:13:41,428 INFO [6ed56fb6cbf7:32901 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T08:13:41,428 INFO [6ed56fb6cbf7:32901 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T08:13:41,428 DEBUG [6ed56fb6cbf7:32901 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T08:13:41,429 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=021eaa39c912c55ccf9b758a5c92d1f8, regionState=OPENING, regionLocation=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,429 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=477e65b3290a1e6e03e5eb59f7aa956d, regionState=OPENING, regionLocation=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,430 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=9eb0364d53b384c3e59ecc15348e6c3c, regionState=OPENING, regionLocation=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,430 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6ed56fb6cbf7:33091 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ed56fb6cbf7/172.17.0.3:33091 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T08:13:41,431 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=3773623806176f3d7f8efb3efe1b7937, regionState=OPENING, regionLocation=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,431 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=f55c025b8995c15117005602a775de64, regionState=OPENING, regionLocation=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ed56fb6cbf7,33091,1732176812968, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ed56fb6cbf7,33091,1732176812968, seqNum=-1, error=java.net.ConnectException: Call to address=6ed56fb6cbf7:33091 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ed56fb6cbf7/172.17.0.3:33091 2024-11-21T08:13:41,432 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=dbfd5210cf46b2a246edeeebd244b662, regionState=OPENING, regionLocation=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,433 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ed56fb6cbf7,33091,1732176812968, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-21T08:13:41,433 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ed56fb6cbf7,33091,1732176812968, seqNum=-1 from cache 2024-11-21T08:13:41,433 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=cd29f056dad8c6c37d0979434891505d, regionState=OPENING, regionLocation=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,433 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T08:13:41,433 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ed56fb6cbf7,35295,1732176812677, seqNum=-1] 2024-11-21T08:13:41,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 6ed56fb6cbf7:33091 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ed56fb6cbf7/172.17.0.3:33091 2024-11-21T08:13:41,434 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T08:13:41,434 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=fd0a67dc199543d2675c547a7cb41702, regionState=OPENING, regionLocation=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,434 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6ed56fb6cbf7:33091 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ed56fb6cbf7/172.17.0.3:33091 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T08:13:41,435 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ed56fb6cbf7,33091,1732176812968, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ed56fb6cbf7,35295,1732176812677, seqNum=-1, error=java.net.ConnectException: Call to address=6ed56fb6cbf7:33091 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ed56fb6cbf7/172.17.0.3:33091 2024-11-21T08:13:41,435 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ed56fb6cbf7,33091,1732176812968, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ed56fb6cbf7,35295,1732176812677, seqNum=-1, error=java.net.ConnectException: Call to address=6ed56fb6cbf7:33091 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ed56fb6cbf7/172.17.0.3:33091 2024-11-21T08:13:41,435 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ed56fb6cbf7,33091,1732176812968, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ed56fb6cbf7,35295,1732176812677, seqNum=-1, error=java.net.ConnectException: Call to address=6ed56fb6cbf7:33091 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ed56fb6cbf7/172.17.0.3:33091 2024-11-21T08:13:41,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ed56fb6cbf7,33091,1732176812968, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ed56fb6cbf7,35295,1732176812677, seqNum=-1, error=java.net.ConnectException: Call to address=6ed56fb6cbf7:33091 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ed56fb6cbf7/172.17.0.3:33091 2024-11-21T08:13:41,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ed56fb6cbf7,33091,1732176812968, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ed56fb6cbf7,35295,1732176812677, seqNum=-1, error=java.net.ConnectException: Call to address=6ed56fb6cbf7:33091 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ed56fb6cbf7/172.17.0.3:33091 2024-11-21T08:13:41,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 6ed56fb6cbf7:33091 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ed56fb6cbf7/172.17.0.3:33091 2024-11-21T08:13:41,436 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40127, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T08:13:41,443 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN because future has completed 2024-11-21T08:13:41,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866}] 2024-11-21T08:13:41,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN because future has completed 2024-11-21T08:13:41,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd0a67dc199543d2675c547a7cb41702, server=6ed56fb6cbf7,38845,1732176812866}] 2024-11-21T08:13:41,541 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN because future has completed 2024-11-21T08:13:41,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 021eaa39c912c55ccf9b758a5c92d1f8, server=6ed56fb6cbf7,35295,1732176812677}] 2024-11-21T08:13:41,545 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN because future has completed 2024-11-21T08:13:41,549 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN because future has completed 2024-11-21T08:13:41,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN because future has completed 2024-11-21T08:13:41,552 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3773623806176f3d7f8efb3efe1b7937, server=6ed56fb6cbf7,35295,1732176812677}] 2024-11-21T08:13:41,554 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN because future has completed 2024-11-21T08:13:41,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN because future has completed 2024-11-21T08:13:41,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure dbfd5210cf46b2a246edeeebd244b662, server=6ed56fb6cbf7,38845,1732176812866}] 2024-11-21T08:13:41,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure f55c025b8995c15117005602a775de64, server=6ed56fb6cbf7,38845,1732176812866}] 2024-11-21T08:13:41,573 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9eb0364d53b384c3e59ecc15348e6c3c, server=6ed56fb6cbf7,35295,1732176812677}] 2024-11-21T08:13:41,576 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 477e65b3290a1e6e03e5eb59f7aa956d, server=6ed56fb6cbf7,35295,1732176812677}] 2024-11-21T08:13:41,602 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,eee,1732176816148.cd29f056dad8c6c37d0979434891505d. 2024-11-21T08:13:41,602 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => cd29f056dad8c6c37d0979434891505d, NAME => 'test,eee,1732176816148.cd29f056dad8c6c37d0979434891505d.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-21T08:13:41,603 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test cd29f056dad8c6c37d0979434891505d 2024-11-21T08:13:41,603 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,eee,1732176816148.cd29f056dad8c6c37d0979434891505d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:41,603 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for cd29f056dad8c6c37d0979434891505d 2024-11-21T08:13:41,603 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for cd29f056dad8c6c37d0979434891505d 2024-11-21T08:13:41,605 INFO [StoreOpener-cd29f056dad8c6c37d0979434891505d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region cd29f056dad8c6c37d0979434891505d 2024-11-21T08:13:41,607 INFO [StoreOpener-cd29f056dad8c6c37d0979434891505d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cd29f056dad8c6c37d0979434891505d columnFamilyName family 2024-11-21T08:13:41,607 DEBUG [StoreOpener-cd29f056dad8c6c37d0979434891505d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:41,608 INFO [StoreOpener-cd29f056dad8c6c37d0979434891505d-1 {}] regionserver.HStore(327): Store=cd29f056dad8c6c37d0979434891505d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:41,608 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for cd29f056dad8c6c37d0979434891505d 2024-11-21T08:13:41,609 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/cd29f056dad8c6c37d0979434891505d 2024-11-21T08:13:41,610 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/cd29f056dad8c6c37d0979434891505d 2024-11-21T08:13:41,611 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for cd29f056dad8c6c37d0979434891505d 2024-11-21T08:13:41,611 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for cd29f056dad8c6c37d0979434891505d 2024-11-21T08:13:41,613 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for cd29f056dad8c6c37d0979434891505d 2024-11-21T08:13:41,614 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened cd29f056dad8c6c37d0979434891505d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66187251, jitterRate=-0.01373310387134552}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:41,614 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cd29f056dad8c6c37d0979434891505d 2024-11-21T08:13:41,614 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for cd29f056dad8c6c37d0979434891505d: Running coprocessor pre-open hook at 1732176821603Writing region info on filesystem at 1732176821603Initializing all the Stores at 1732176821605 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176821605Cleaning up temporary data from old regions at 1732176821611 (+6 ms)Running coprocessor post-open hooks at 1732176821614 (+3 ms)Region opened successfully at 1732176821614 2024-11-21T08:13:41,615 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732176816148.cd29f056dad8c6c37d0979434891505d., pid=75, masterSystemTime=1732176821597 2024-11-21T08:13:41,617 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732176816148.cd29f056dad8c6c37d0979434891505d. 2024-11-21T08:13:41,617 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,eee,1732176816148.cd29f056dad8c6c37d0979434891505d. 2024-11-21T08:13:41,618 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,,1732176816148.fd0a67dc199543d2675c547a7cb41702. 2024-11-21T08:13:41,618 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => fd0a67dc199543d2675c547a7cb41702, NAME => 'test,,1732176816148.fd0a67dc199543d2675c547a7cb41702.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-21T08:13:41,618 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fd0a67dc199543d2675c547a7cb41702 2024-11-21T08:13:41,618 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,,1732176816148.fd0a67dc199543d2675c547a7cb41702.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:41,618 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for fd0a67dc199543d2675c547a7cb41702 2024-11-21T08:13:41,618 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for fd0a67dc199543d2675c547a7cb41702 2024-11-21T08:13:41,625 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-21T08:13:41,625 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,625 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:41,627 INFO [StoreOpener-fd0a67dc199543d2675c547a7cb41702-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fd0a67dc199543d2675c547a7cb41702 2024-11-21T08:13:41,634 INFO [StoreOpener-fd0a67dc199543d2675c547a7cb41702-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd0a67dc199543d2675c547a7cb41702 columnFamilyName family 2024-11-21T08:13:41,634 DEBUG [StoreOpener-fd0a67dc199543d2675c547a7cb41702-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:41,635 INFO [StoreOpener-fd0a67dc199543d2675c547a7cb41702-1 {}] regionserver.HStore(327): Store=fd0a67dc199543d2675c547a7cb41702/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:41,635 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for fd0a67dc199543d2675c547a7cb41702 2024-11-21T08:13:41,637 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/fd0a67dc199543d2675c547a7cb41702 2024-11-21T08:13:41,639 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/fd0a67dc199543d2675c547a7cb41702 2024-11-21T08:13:41,640 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for fd0a67dc199543d2675c547a7cb41702 2024-11-21T08:13:41,641 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for fd0a67dc199543d2675c547a7cb41702 2024-11-21T08:13:41,643 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for fd0a67dc199543d2675c547a7cb41702 2024-11-21T08:13:41,656 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened fd0a67dc199543d2675c547a7cb41702; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71912364, jitterRate=0.07157772779464722}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:41,657 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fd0a67dc199543d2675c547a7cb41702 2024-11-21T08:13:41,657 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for fd0a67dc199543d2675c547a7cb41702: Running coprocessor pre-open hook at 1732176821618Writing region info on filesystem at 1732176821618Initializing all the Stores at 1732176821620 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176821620Cleaning up temporary data from old regions at 1732176821641 (+21 ms)Running coprocessor post-open hooks at 1732176821657 (+16 ms)Region opened successfully at 1732176821657 2024-11-21T08:13:41,662 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1732176816148.fd0a67dc199543d2675c547a7cb41702., pid=76, masterSystemTime=1732176821597 2024-11-21T08:13:41,667 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1732176816148.fd0a67dc199543d2675c547a7cb41702. 2024-11-21T08:13:41,667 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,,1732176816148.fd0a67dc199543d2675c547a7cb41702. 2024-11-21T08:13:41,667 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open test,rrr,1732176816148.f55c025b8995c15117005602a775de64. 2024-11-21T08:13:41,668 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => f55c025b8995c15117005602a775de64, NAME => 'test,rrr,1732176816148.f55c025b8995c15117005602a775de64.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-21T08:13:41,666 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-11-21T08:13:41,668 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f55c025b8995c15117005602a775de64 2024-11-21T08:13:41,672 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated test,rrr,1732176816148.f55c025b8995c15117005602a775de64.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:41,672 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for f55c025b8995c15117005602a775de64 2024-11-21T08:13:41,672 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for f55c025b8995c15117005602a775de64 2024-11-21T08:13:41,675 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T08:13:41,675 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 477e65b3290a1e6e03e5eb59f7aa956d, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,675 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:41,675 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 477e65b3290a1e6e03e5eb59f7aa956d, server=6ed56fb6cbf7,35295,1732176812677 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:41,676 INFO [StoreOpener-f55c025b8995c15117005602a775de64-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f55c025b8995c15117005602a775de64 2024-11-21T08:13:41,678 INFO [StoreOpener-f55c025b8995c15117005602a775de64-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f55c025b8995c15117005602a775de64 columnFamilyName family 2024-11-21T08:13:41,678 DEBUG [StoreOpener-f55c025b8995c15117005602a775de64-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:41,679 INFO [StoreOpener-f55c025b8995c15117005602a775de64-1 {}] regionserver.HStore(327): Store=f55c025b8995c15117005602a775de64/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:41,679 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for f55c025b8995c15117005602a775de64 2024-11-21T08:13:41,680 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/f55c025b8995c15117005602a775de64 2024-11-21T08:13:41,682 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/f55c025b8995c15117005602a775de64 2024-11-21T08:13:41,682 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for f55c025b8995c15117005602a775de64 2024-11-21T08:13:41,682 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for f55c025b8995c15117005602a775de64 2024-11-21T08:13:41,684 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for f55c025b8995c15117005602a775de64 2024-11-21T08:13:41,685 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened f55c025b8995c15117005602a775de64; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65578091, jitterRate=-0.02281029522418976}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:41,685 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f55c025b8995c15117005602a775de64 2024-11-21T08:13:41,686 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for f55c025b8995c15117005602a775de64: Running coprocessor pre-open hook at 1732176821672Writing region info on filesystem at 1732176821672Initializing all the Stores at 1732176821674 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176821674Cleaning up temporary data from old regions at 1732176821682 (+8 ms)Running coprocessor post-open hooks at 1732176821685 (+3 ms)Region opened successfully at 1732176821685 2024-11-21T08:13:41,686 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732176816148.f55c025b8995c15117005602a775de64., pid=80, masterSystemTime=1732176821597 2024-11-21T08:13:41,689 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732176816148.f55c025b8995c15117005602a775de64. 2024-11-21T08:13:41,689 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened test,rrr,1732176816148.f55c025b8995c15117005602a775de64. 2024-11-21T08:13:41,689 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,kkk,1732176816148.dbfd5210cf46b2a246edeeebd244b662. 2024-11-21T08:13:41,690 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => dbfd5210cf46b2a246edeeebd244b662, NAME => 'test,kkk,1732176816148.dbfd5210cf46b2a246edeeebd244b662.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-21T08:13:41,690 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test dbfd5210cf46b2a246edeeebd244b662 2024-11-21T08:13:41,690 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,kkk,1732176816148.dbfd5210cf46b2a246edeeebd244b662.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:41,690 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for dbfd5210cf46b2a246edeeebd244b662 2024-11-21T08:13:41,690 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for dbfd5210cf46b2a246edeeebd244b662 2024-11-21T08:13:41,692 INFO [StoreOpener-dbfd5210cf46b2a246edeeebd244b662-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region dbfd5210cf46b2a246edeeebd244b662 2024-11-21T08:13:41,693 INFO [StoreOpener-dbfd5210cf46b2a246edeeebd244b662-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbfd5210cf46b2a246edeeebd244b662 columnFamilyName family 2024-11-21T08:13:41,694 DEBUG [StoreOpener-dbfd5210cf46b2a246edeeebd244b662-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:41,694 INFO [StoreOpener-dbfd5210cf46b2a246edeeebd244b662-1 {}] regionserver.HStore(327): Store=dbfd5210cf46b2a246edeeebd244b662/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:41,694 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for dbfd5210cf46b2a246edeeebd244b662 2024-11-21T08:13:41,695 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/dbfd5210cf46b2a246edeeebd244b662 2024-11-21T08:13:41,697 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/dbfd5210cf46b2a246edeeebd244b662 2024-11-21T08:13:41,697 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for dbfd5210cf46b2a246edeeebd244b662 2024-11-21T08:13:41,697 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for dbfd5210cf46b2a246edeeebd244b662 2024-11-21T08:13:41,698 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T08:13:41,698 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:41,698 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T08:13:41,698 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T08:13:41,699 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:41,700 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T08:13:41,700 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T08:13:41,700 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T08:13:41,700 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T08:13:41,705 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for dbfd5210cf46b2a246edeeebd244b662 2024-11-21T08:13:41,705 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open test,hhh,1732176816148.9eb0364d53b384c3e59ecc15348e6c3c. 2024-11-21T08:13:41,705 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => 9eb0364d53b384c3e59ecc15348e6c3c, NAME => 'test,hhh,1732176816148.9eb0364d53b384c3e59ecc15348e6c3c.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-21T08:13:41,706 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9eb0364d53b384c3e59ecc15348e6c3c 2024-11-21T08:13:41,706 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated test,hhh,1732176816148.9eb0364d53b384c3e59ecc15348e6c3c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T08:13:41,706 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for 9eb0364d53b384c3e59ecc15348e6c3c 2024-11-21T08:13:41,706 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for 9eb0364d53b384c3e59ecc15348e6c3c 2024-11-21T08:13:41,707 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened dbfd5210cf46b2a246edeeebd244b662; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70988566, jitterRate=0.05781206488609314}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:41,707 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dbfd5210cf46b2a246edeeebd244b662 2024-11-21T08:13:41,707 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 6 msec 2024-11-21T08:13:41,707 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for dbfd5210cf46b2a246edeeebd244b662: Running coprocessor pre-open hook at 1732176821690Writing region info on filesystem at 1732176821690Initializing all the Stores at 1732176821692 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176821692Cleaning up temporary data from old regions at 1732176821697 (+5 ms)Running coprocessor post-open hooks at 1732176821707 (+10 ms)Region opened successfully at 1732176821707 2024-11-21T08:13:41,709 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:41,709 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1732176816148.dbfd5210cf46b2a246edeeebd244b662., pid=79, masterSystemTime=1732176821597 2024-11-21T08:13:41,710 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:41,712 INFO [StoreOpener-9eb0364d53b384c3e59ecc15348e6c3c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9eb0364d53b384c3e59ecc15348e6c3c 2024-11-21T08:13:41,714 INFO [StoreOpener-9eb0364d53b384c3e59ecc15348e6c3c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9eb0364d53b384c3e59ecc15348e6c3c columnFamilyName family 2024-11-21T08:13:41,714 DEBUG [StoreOpener-9eb0364d53b384c3e59ecc15348e6c3c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T08:13:41,715 INFO [StoreOpener-9eb0364d53b384c3e59ecc15348e6c3c-1 {}] regionserver.HStore(327): Store=9eb0364d53b384c3e59ecc15348e6c3c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T08:13:41,715 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for 9eb0364d53b384c3e59ecc15348e6c3c 2024-11-21T08:13:41,716 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/9eb0364d53b384c3e59ecc15348e6c3c 2024-11-21T08:13:41,717 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T08:13:41,718 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/9eb0364d53b384c3e59ecc15348e6c3c 2024-11-21T08:13:41,718 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:41,718 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:41,718 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,718 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,718 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,718 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,718 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,718 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for 9eb0364d53b384c3e59ecc15348e6c3c 2024-11-21T08:13:41,718 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=82, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,718 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for 9eb0364d53b384c3e59ecc15348e6c3c 2024-11-21T08:13:41,718 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,718 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,719 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,719 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,719 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,719 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,719 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,719 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,719 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,719 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,719 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:41,719 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:41,719 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,719 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,720 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T08:13:41,721 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for 9eb0364d53b384c3e59ecc15348e6c3c 2024-11-21T08:13:41,722 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T08:13:41,722 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened 9eb0364d53b384c3e59ecc15348e6c3c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72125089, jitterRate=0.07474757730960846}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T08:13:41,722 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9eb0364d53b384c3e59ecc15348e6c3c 2024-11-21T08:13:41,722 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, REOPEN/MOVE 2024-11-21T08:13:41,723 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for 9eb0364d53b384c3e59ecc15348e6c3c: Running coprocessor pre-open hook at 1732176821706Writing region info on filesystem at 1732176821706Initializing all the Stores at 1732176821708 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732176821709 (+1 ms)Cleaning up temporary data from old regions at 1732176821718 (+9 ms)Running coprocessor post-open hooks at 1732176821722 (+4 ms)Region opened successfully at 1732176821722 2024-11-21T08:13:41,724 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1732176816148.9eb0364d53b384c3e59ecc15348e6c3c., pid=81, masterSystemTime=1732176821698 2024-11-21T08:13:41,725 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:41,730 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true 2024-11-21T08:13:41,730 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T08:13:41,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:41,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:41,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:41,730 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "6ed56fb6cbf7" port: 38845 start_code: 1732176812866 } transition { transition_code: OPENED region_info { region_id: 1732176816148 table_name { namespace: "default" qualifier: "test" } start_key: "kkk" end_key: "lll" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 79 initiating_master_active_time: 1732176813431 }; retry (#0) after 1000ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-21T08:13:41,732 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN 2024-11-21T08:13:41,732 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN 2024-11-21T08:13:41,732 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN 2024-11-21T08:13:41,732 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN 2024-11-21T08:13:41,732 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN 2024-11-21T08:13:41,732 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,732 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd0a67dc199543d2675c547a7cb41702, server=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,732 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 021eaa39c912c55ccf9b758a5c92d1f8, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,732 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3773623806176f3d7f8efb3efe1b7937, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,732 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure dbfd5210cf46b2a246edeeebd244b662, server=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,732 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure f55c025b8995c15117005602a775de64, server=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,732 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9eb0364d53b384c3e59ecc15348e6c3c, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,732 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 477e65b3290a1e6e03e5eb59f7aa956d, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,733 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,733 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,733 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,733 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,733 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,733 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,733 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,732 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "6ed56fb6cbf7" port: 35295 start_code: 1732176812677 } transition { transition_code: OPENED region_info { region_id: 1732176816148 table_name { namespace: "default" qualifier: "test" } start_key: "hhh" end_key: "iii" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 81 initiating_master_active_time: 1732176813431 }; retry (#0) after 1004ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-21T08:13:41,733 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,737 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:41,737 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,737 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:41,737 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,737 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:41,737 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,737 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN 2024-11-21T08:13:41,737 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,737 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN 2024-11-21T08:13:41,737 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,737 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN 2024-11-21T08:13:41,737 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,738 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN 2024-11-21T08:13:41,738 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,738 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN 2024-11-21T08:13:41,738 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,738 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd0a67dc199543d2675c547a7cb41702, server=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,738 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 021eaa39c912c55ccf9b758a5c92d1f8, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,738 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3773623806176f3d7f8efb3efe1b7937, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,738 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure dbfd5210cf46b2a246edeeebd244b662, server=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,738 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure f55c025b8995c15117005602a775de64, server=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,738 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9eb0364d53b384c3e59ecc15348e6c3c, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,738 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 477e65b3290a1e6e03e5eb59f7aa956d, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,739 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 32 msec 2024-11-21T08:13:41,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=65 2024-11-21T08:13:41,742 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=hbase:meta, region=1588230740 2024-11-21T08:13:41,743 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ed56fb6cbf7,35295,1732176812677, state=OPEN 2024-11-21T08:13:41,761 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:41,761 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:41,761 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:41,762 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:41,762 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:41,762 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:41,763 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN to state=OFFLINE, location=null, table=test, region=021eaa39c912c55ccf9b758a5c92d1f8 to restore RIT 2024-11-21T08:13:41,763 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN to state=OFFLINE, location=null, table=test, region=cd29f056dad8c6c37d0979434891505d to restore RIT 2024-11-21T08:13:41,763 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN to state=OFFLINE, location=null, table=test, region=477e65b3290a1e6e03e5eb59f7aa956d to restore RIT 2024-11-21T08:13:41,763 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN to state=OFFLINE, location=null, table=test, region=dbfd5210cf46b2a246edeeebd244b662 to restore RIT 2024-11-21T08:13:41,763 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN to state=OFFLINE, location=null, table=test, region=fd0a67dc199543d2675c547a7cb41702 to restore RIT 2024-11-21T08:13:41,763 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN to state=OFFLINE, location=null, table=test, region=f55c025b8995c15117005602a775de64 to restore RIT 2024-11-21T08:13:41,763 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN to state=OFFLINE, location=null, table=test, region=3773623806176f3d7f8efb3efe1b7937 to restore RIT 2024-11-21T08:13:41,763 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN to state=OFFLINE, location=null, table=test, region=9eb0364d53b384c3e59ecc15348e6c3c to restore RIT 2024-11-21T08:13:41,764 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732176851764 2024-11-21T08:13:41,764 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T08:13:41,764 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:41,766 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T08:13:41,768 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T08:13:41,768 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9eb0364d53b384c3e59ecc15348e6c3c, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:41,769 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9eb0364d53b384c3e59ecc15348e6c3c, server=6ed56fb6cbf7,35295,1732176812677 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:41,800 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd0a67dc199543d2675c547a7cb41702, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,800 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3773623806176f3d7f8efb3efe1b7937, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,801 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f38fd8b8defc002b7cb3de558717f27c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,801 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f43187c46588bff73d01ba7eb26a6f8, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,801 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd29f056dad8c6c37d0979434891505d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=829529fdccf9853d96113fa67713ffbc, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee7ef14c3df8915e550cfda528563489, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9eb0364d53b384c3e59ecc15348e6c3c, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd97d8f813bdfcba3626cc4950c7c1e1, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5fb158c8201cfbc5babdbfc33ee45ab1, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbfd5210cf46b2a246edeeebd244b662, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94c604fd0d435e28ac643b49d6cbee14, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d72691fc99dffac743de3d6bcb0f84ed, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99e7b3565f08f10e8c368072e9c21c09, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=477e65b3290a1e6e03e5eb59f7aa956d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5017ae3acb74fe8c7d0b9c6d9a2cc402, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d5b1f002057972e76beae8f19094030, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f55c025b8995c15117005602a775de64, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed3111e35326fc2c5455ae2475ab1b5, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec3a5c93577675c61d946ed41b85bdb5, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ef5fca53c8ff0d12fc112ab68731575, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=5 2024-11-21T08:13:41,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dfe569750c256f741c6d3e5ed072503, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8202a2e0e04a2c5c7f5f0e491d40e, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=021eaa39c912c55ccf9b758a5c92d1f8, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af43494f9c57dd49b6d504756610f96c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3589ece79012bcc3080c57b249902675, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,807 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T08:13:41,807 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732176881807 2024-11-21T08:13:41,807 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732176941807 2024-11-21T08:13:41,807 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 43 msec 2024-11-21T08:13:41,808 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T08:13:41,808 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:41,808 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T08:13:41,808 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T08:13:41,808 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T08:13:41,808 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T08:13:41,808 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T08:13:41,809 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T08:13:41,809 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T08:13:41,816 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T08:13:41,816 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:41,816 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:41,816 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,816 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,816 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,816 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,816 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,816 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,816 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,816 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,816 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,816 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,816 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,816 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,817 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,817 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,817 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,817 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:41,817 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:41,817 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,817 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,817 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T08:13:41,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T08:13:41,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, REOPEN/MOVE 2024-11-21T08:13:41,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true 2024-11-21T08:13:41,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T08:13:41,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:41,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:41,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:41,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN 2024-11-21T08:13:41,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN 2024-11-21T08:13:41,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN 2024-11-21T08:13:41,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN 2024-11-21T08:13:41,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN 2024-11-21T08:13:41,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd0a67dc199543d2675c547a7cb41702, server=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 021eaa39c912c55ccf9b758a5c92d1f8, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3773623806176f3d7f8efb3efe1b7937, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure dbfd5210cf46b2a246edeeebd244b662, server=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure f55c025b8995c15117005602a775de64, server=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9eb0364d53b384c3e59ecc15348e6c3c, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,824 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,825 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,825 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,825 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,825 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,825 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,825 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,825 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,825 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:41,825 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,825 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:41,825 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,825 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:41,825 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,825 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN 2024-11-21T08:13:41,825 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,826 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN 2024-11-21T08:13:41,826 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,826 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN 2024-11-21T08:13:41,826 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,826 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN 2024-11-21T08:13:41,826 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,826 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN 2024-11-21T08:13:41,826 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,826 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd0a67dc199543d2675c547a7cb41702, server=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,826 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 021eaa39c912c55ccf9b758a5c92d1f8, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,826 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3773623806176f3d7f8efb3efe1b7937, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,826 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure dbfd5210cf46b2a246edeeebd244b662, server=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,826 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure f55c025b8995c15117005602a775de64, server=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,826 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9eb0364d53b384c3e59ecc15348e6c3c, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,826 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 16 msec 2024-11-21T08:13:41,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=65 2024-11-21T08:13:41,828 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=hbase:meta, region=1588230740 2024-11-21T08:13:41,828 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ed56fb6cbf7,35295,1732176812677, state=OPEN 2024-11-21T08:13:41,833 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:41,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:41,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:41,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:41,835 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:41,835 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:41,835 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:41,836 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN to state=OFFLINE, location=null, table=test, region=021eaa39c912c55ccf9b758a5c92d1f8 to restore RIT 2024-11-21T08:13:41,836 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN to state=OFFLINE, location=null, table=test, region=cd29f056dad8c6c37d0979434891505d to restore RIT 2024-11-21T08:13:41,836 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN to state=OFFLINE, location=null, table=test, region=477e65b3290a1e6e03e5eb59f7aa956d to restore RIT 2024-11-21T08:13:41,836 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN to state=OFFLINE, location=null, table=test, region=dbfd5210cf46b2a246edeeebd244b662 to restore RIT 2024-11-21T08:13:41,837 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN to state=OFFLINE, location=null, table=test, region=fd0a67dc199543d2675c547a7cb41702 to restore RIT 2024-11-21T08:13:41,837 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN to state=OFFLINE, location=null, table=test, region=f55c025b8995c15117005602a775de64 to restore RIT 2024-11-21T08:13:41,837 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN to state=OFFLINE, location=null, table=test, region=3773623806176f3d7f8efb3efe1b7937 to restore RIT 2024-11-21T08:13:41,837 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN to state=OFFLINE, location=null, table=test, region=9eb0364d53b384c3e59ecc15348e6c3c to restore RIT 2024-11-21T08:13:41,837 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732176851837 2024-11-21T08:13:41,837 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T08:13:41,837 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:41,839 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T08:13:41,840 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T08:13:41,840 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=80, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure f55c025b8995c15117005602a775de64, server=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,840 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:41,840 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=80, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure f55c025b8995c15117005602a775de64, server=6ed56fb6cbf7,38845,1732176812866 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:41,849 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd0a67dc199543d2675c547a7cb41702, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,849 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3773623806176f3d7f8efb3efe1b7937, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,849 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f38fd8b8defc002b7cb3de558717f27c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,849 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f43187c46588bff73d01ba7eb26a6f8, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,849 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd29f056dad8c6c37d0979434891505d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,850 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=829529fdccf9853d96113fa67713ffbc, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,850 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee7ef14c3df8915e550cfda528563489, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,850 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9eb0364d53b384c3e59ecc15348e6c3c, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,850 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd97d8f813bdfcba3626cc4950c7c1e1, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,850 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5fb158c8201cfbc5babdbfc33ee45ab1, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,850 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbfd5210cf46b2a246edeeebd244b662, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,850 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94c604fd0d435e28ac643b49d6cbee14, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d72691fc99dffac743de3d6bcb0f84ed, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99e7b3565f08f10e8c368072e9c21c09, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=477e65b3290a1e6e03e5eb59f7aa956d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5017ae3acb74fe8c7d0b9c6d9a2cc402, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d5b1f002057972e76beae8f19094030, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f55c025b8995c15117005602a775de64, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed3111e35326fc2c5455ae2475ab1b5, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec3a5c93577675c61d946ed41b85bdb5, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ef5fca53c8ff0d12fc112ab68731575, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=5 2024-11-21T08:13:41,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dfe569750c256f741c6d3e5ed072503, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8202a2e0e04a2c5c7f5f0e491d40e, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=021eaa39c912c55ccf9b758a5c92d1f8, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af43494f9c57dd49b6d504756610f96c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3589ece79012bcc3080c57b249902675, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,853 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T08:13:41,853 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732176881853 2024-11-21T08:13:41,853 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732176941853 2024-11-21T08:13:41,853 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 15 msec 2024-11-21T08:13:41,853 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T08:13:41,853 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:41,853 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T08:13:41,853 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T08:13:41,853 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T08:13:41,853 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T08:13:41,853 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T08:13:41,854 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T08:13:41,855 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T08:13:41,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T08:13:41,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:41,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:41,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:41,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:41,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T08:13:41,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T08:13:41,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, REOPEN/MOVE 2024-11-21T08:13:41,864 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true 2024-11-21T08:13:41,865 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,865 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,865 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T08:13:41,865 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,865 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,865 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,865 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:41,865 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:41,865 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:41,865 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN 2024-11-21T08:13:41,865 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN 2024-11-21T08:13:41,865 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN 2024-11-21T08:13:41,865 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN 2024-11-21T08:13:41,865 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN 2024-11-21T08:13:41,865 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,865 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd0a67dc199543d2675c547a7cb41702, server=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,865 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 021eaa39c912c55ccf9b758a5c92d1f8, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,865 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3773623806176f3d7f8efb3efe1b7937, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,865 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure dbfd5210cf46b2a246edeeebd244b662, server=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure f55c025b8995c15117005602a775de64, server=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,866 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,866 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,866 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,866 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,866 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,866 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,866 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,866 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,866 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:41,866 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,866 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:41,866 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,866 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:41,866 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,866 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN 2024-11-21T08:13:41,866 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,866 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN 2024-11-21T08:13:41,866 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,866 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN 2024-11-21T08:13:41,866 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,866 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN 2024-11-21T08:13:41,866 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,866 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN 2024-11-21T08:13:41,867 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,867 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd0a67dc199543d2675c547a7cb41702, server=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,867 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 021eaa39c912c55ccf9b758a5c92d1f8, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,867 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3773623806176f3d7f8efb3efe1b7937, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,867 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure dbfd5210cf46b2a246edeeebd244b662, server=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,867 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure f55c025b8995c15117005602a775de64, server=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,867 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 12 msec 2024-11-21T08:13:41,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=65 2024-11-21T08:13:41,868 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=hbase:meta, region=1588230740 2024-11-21T08:13:41,869 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ed56fb6cbf7,35295,1732176812677, state=OPEN 2024-11-21T08:13:41,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:41,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:41,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:41,877 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:41,877 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:41,877 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:41,878 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN to state=OFFLINE, location=null, table=test, region=021eaa39c912c55ccf9b758a5c92d1f8 to restore RIT 2024-11-21T08:13:41,879 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN to state=OFFLINE, location=null, table=test, region=cd29f056dad8c6c37d0979434891505d to restore RIT 2024-11-21T08:13:41,879 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN to state=OFFLINE, location=null, table=test, region=477e65b3290a1e6e03e5eb59f7aa956d to restore RIT 2024-11-21T08:13:41,879 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN to state=OFFLINE, location=null, table=test, region=dbfd5210cf46b2a246edeeebd244b662 to restore RIT 2024-11-21T08:13:41,879 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN to state=OFFLINE, location=null, table=test, region=fd0a67dc199543d2675c547a7cb41702 to restore RIT 2024-11-21T08:13:41,879 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN to state=OFFLINE, location=null, table=test, region=f55c025b8995c15117005602a775de64 to restore RIT 2024-11-21T08:13:41,879 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN to state=OFFLINE, location=null, table=test, region=3773623806176f3d7f8efb3efe1b7937 to restore RIT 2024-11-21T08:13:41,879 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN to state=OFFLINE, location=null, table=test, region=9eb0364d53b384c3e59ecc15348e6c3c to restore RIT 2024-11-21T08:13:41,879 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732176851879 2024-11-21T08:13:41,879 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T08:13:41,879 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:41,881 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T08:13:41,882 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T08:13:41,882 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure dbfd5210cf46b2a246edeeebd244b662, server=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:41,882 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure dbfd5210cf46b2a246edeeebd244b662, server=6ed56fb6cbf7,38845,1732176812866 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:41,889 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd0a67dc199543d2675c547a7cb41702, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,889 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3773623806176f3d7f8efb3efe1b7937, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,889 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f38fd8b8defc002b7cb3de558717f27c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f43187c46588bff73d01ba7eb26a6f8, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd29f056dad8c6c37d0979434891505d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=829529fdccf9853d96113fa67713ffbc, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee7ef14c3df8915e550cfda528563489, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9eb0364d53b384c3e59ecc15348e6c3c, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd97d8f813bdfcba3626cc4950c7c1e1, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5fb158c8201cfbc5babdbfc33ee45ab1, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbfd5210cf46b2a246edeeebd244b662, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94c604fd0d435e28ac643b49d6cbee14, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,892 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d72691fc99dffac743de3d6bcb0f84ed, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,892 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99e7b3565f08f10e8c368072e9c21c09, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,892 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=477e65b3290a1e6e03e5eb59f7aa956d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,892 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5017ae3acb74fe8c7d0b9c6d9a2cc402, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,892 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d5b1f002057972e76beae8f19094030, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f55c025b8995c15117005602a775de64, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed3111e35326fc2c5455ae2475ab1b5, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec3a5c93577675c61d946ed41b85bdb5, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ef5fca53c8ff0d12fc112ab68731575, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=5 2024-11-21T08:13:41,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dfe569750c256f741c6d3e5ed072503, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8202a2e0e04a2c5c7f5f0e491d40e, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=021eaa39c912c55ccf9b758a5c92d1f8, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af43494f9c57dd49b6d504756610f96c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3589ece79012bcc3080c57b249902675, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,894 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T08:13:41,894 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732176881894 2024-11-21T08:13:41,895 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732176941895 2024-11-21T08:13:41,895 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 15 msec 2024-11-21T08:13:41,895 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T08:13:41,895 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:41,895 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T08:13:41,895 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T08:13:41,895 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T08:13:41,895 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T08:13:41,895 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T08:13:41,896 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T08:13:41,896 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T08:13:41,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T08:13:41,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:41,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:41,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:41,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:41,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,903 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T08:13:41,904 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T08:13:41,904 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, REOPEN/MOVE 2024-11-21T08:13:41,907 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true 2024-11-21T08:13:41,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T08:13:41,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:41,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:41,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:41,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN 2024-11-21T08:13:41,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN 2024-11-21T08:13:41,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN 2024-11-21T08:13:41,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN 2024-11-21T08:13:41,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN 2024-11-21T08:13:41,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd0a67dc199543d2675c547a7cb41702, server=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 021eaa39c912c55ccf9b758a5c92d1f8, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3773623806176f3d7f8efb3efe1b7937, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure dbfd5210cf46b2a246edeeebd244b662, server=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,909 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,909 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,909 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,909 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,909 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,909 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,909 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,909 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,909 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:41,909 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,909 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:41,909 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,909 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:41,909 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,909 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN 2024-11-21T08:13:41,909 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,910 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN 2024-11-21T08:13:41,910 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,910 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN 2024-11-21T08:13:41,910 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,910 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN 2024-11-21T08:13:41,910 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,910 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN 2024-11-21T08:13:41,910 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,910 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd0a67dc199543d2675c547a7cb41702, server=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,910 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 021eaa39c912c55ccf9b758a5c92d1f8, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,910 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3773623806176f3d7f8efb3efe1b7937, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,910 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure dbfd5210cf46b2a246edeeebd244b662, server=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,910 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-11-21T08:13:41,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=65 2024-11-21T08:13:41,912 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=hbase:meta, region=1588230740 2024-11-21T08:13:41,912 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ed56fb6cbf7,35295,1732176812677, state=OPEN 2024-11-21T08:13:41,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:41,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:41,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:41,919 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:41,920 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:41,920 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:41,922 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN to state=OFFLINE, location=null, table=test, region=021eaa39c912c55ccf9b758a5c92d1f8 to restore RIT 2024-11-21T08:13:41,922 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN to state=OFFLINE, location=null, table=test, region=cd29f056dad8c6c37d0979434891505d to restore RIT 2024-11-21T08:13:41,922 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN to state=OFFLINE, location=null, table=test, region=477e65b3290a1e6e03e5eb59f7aa956d to restore RIT 2024-11-21T08:13:41,922 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN to state=OFFLINE, location=null, table=test, region=dbfd5210cf46b2a246edeeebd244b662 to restore RIT 2024-11-21T08:13:41,922 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN to state=OFFLINE, location=null, table=test, region=fd0a67dc199543d2675c547a7cb41702 to restore RIT 2024-11-21T08:13:41,922 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN to state=OFFLINE, location=null, table=test, region=f55c025b8995c15117005602a775de64 to restore RIT 2024-11-21T08:13:41,922 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN to state=OFFLINE, location=null, table=test, region=3773623806176f3d7f8efb3efe1b7937 to restore RIT 2024-11-21T08:13:41,922 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN to state=OFFLINE, location=null, table=test, region=9eb0364d53b384c3e59ecc15348e6c3c to restore RIT 2024-11-21T08:13:41,924 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732176851924 2024-11-21T08:13:41,924 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T08:13:41,924 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:41,926 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T08:13:41,944 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T08:13:41,944 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3773623806176f3d7f8efb3efe1b7937, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:41,945 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3773623806176f3d7f8efb3efe1b7937, server=6ed56fb6cbf7,35295,1732176812677 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:41,951 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd0a67dc199543d2675c547a7cb41702, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,951 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3773623806176f3d7f8efb3efe1b7937, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f38fd8b8defc002b7cb3de558717f27c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f43187c46588bff73d01ba7eb26a6f8, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd29f056dad8c6c37d0979434891505d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=829529fdccf9853d96113fa67713ffbc, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee7ef14c3df8915e550cfda528563489, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9eb0364d53b384c3e59ecc15348e6c3c, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd97d8f813bdfcba3626cc4950c7c1e1, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5fb158c8201cfbc5babdbfc33ee45ab1, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbfd5210cf46b2a246edeeebd244b662, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94c604fd0d435e28ac643b49d6cbee14, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d72691fc99dffac743de3d6bcb0f84ed, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99e7b3565f08f10e8c368072e9c21c09, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,955 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=477e65b3290a1e6e03e5eb59f7aa956d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,955 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5017ae3acb74fe8c7d0b9c6d9a2cc402, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,955 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d5b1f002057972e76beae8f19094030, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,955 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f55c025b8995c15117005602a775de64, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,956 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed3111e35326fc2c5455ae2475ab1b5, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,956 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec3a5c93577675c61d946ed41b85bdb5, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,956 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ef5fca53c8ff0d12fc112ab68731575, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=5 2024-11-21T08:13:41,956 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dfe569750c256f741c6d3e5ed072503, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,956 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8202a2e0e04a2c5c7f5f0e491d40e, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,957 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=021eaa39c912c55ccf9b758a5c92d1f8, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,957 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af43494f9c57dd49b6d504756610f96c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:41,957 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3589ece79012bcc3080c57b249902675, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:41,957 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T08:13:41,957 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732176881957 2024-11-21T08:13:41,957 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732176941957 2024-11-21T08:13:41,958 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 33 msec 2024-11-21T08:13:41,958 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T08:13:41,958 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:41,958 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T08:13:41,958 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T08:13:41,961 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T08:13:41,961 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T08:13:41,961 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T08:13:41,961 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T08:13:41,964 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 3 msec 2024-11-21T08:13:41,968 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T08:13:41,969 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:41,969 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:41,969 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,969 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,969 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,969 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,969 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,969 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,969 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,969 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,969 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,969 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,969 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,969 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,969 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:41,969 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:41,969 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:41,969 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:41,970 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T08:13:41,971 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T08:13:41,971 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, REOPEN/MOVE 2024-11-21T08:13:41,974 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true 2024-11-21T08:13:41,974 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,974 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,974 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T08:13:41,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:41,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:41,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:41,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN 2024-11-21T08:13:41,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN 2024-11-21T08:13:41,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN 2024-11-21T08:13:41,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN 2024-11-21T08:13:41,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN 2024-11-21T08:13:41,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd0a67dc199543d2675c547a7cb41702, server=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:41,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 021eaa39c912c55ccf9b758a5c92d1f8, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3773623806176f3d7f8efb3efe1b7937, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:41,975 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,975 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,976 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,976 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,976 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,976 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,976 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,976 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,976 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:41,976 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,976 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:41,976 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,976 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:41,976 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,976 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN 2024-11-21T08:13:41,976 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,976 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN 2024-11-21T08:13:41,976 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,976 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN 2024-11-21T08:13:41,976 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,977 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN 2024-11-21T08:13:41,977 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:41,977 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN 2024-11-21T08:13:41,977 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,977 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd0a67dc199543d2675c547a7cb41702, server=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,977 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 021eaa39c912c55ccf9b758a5c92d1f8, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,977 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3773623806176f3d7f8efb3efe1b7937, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:41,977 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 12 msec 2024-11-21T08:13:41,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=65 2024-11-21T08:13:41,981 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=hbase:meta, region=1588230740 2024-11-21T08:13:41,981 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ed56fb6cbf7,35295,1732176812677, state=OPEN 2024-11-21T08:13:41,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:41,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:41,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:41,993 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:41,993 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:41,993 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:41,995 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN to state=OFFLINE, location=null, table=test, region=021eaa39c912c55ccf9b758a5c92d1f8 to restore RIT 2024-11-21T08:13:41,995 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN to state=OFFLINE, location=null, table=test, region=cd29f056dad8c6c37d0979434891505d to restore RIT 2024-11-21T08:13:41,995 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN to state=OFFLINE, location=null, table=test, region=477e65b3290a1e6e03e5eb59f7aa956d to restore RIT 2024-11-21T08:13:41,995 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN to state=OFFLINE, location=null, table=test, region=dbfd5210cf46b2a246edeeebd244b662 to restore RIT 2024-11-21T08:13:41,995 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN to state=OFFLINE, location=null, table=test, region=fd0a67dc199543d2675c547a7cb41702 to restore RIT 2024-11-21T08:13:41,995 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN to state=OFFLINE, location=null, table=test, region=f55c025b8995c15117005602a775de64 to restore RIT 2024-11-21T08:13:41,995 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN to state=OFFLINE, location=null, table=test, region=3773623806176f3d7f8efb3efe1b7937 to restore RIT 2024-11-21T08:13:41,995 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN to state=OFFLINE, location=null, table=test, region=9eb0364d53b384c3e59ecc15348e6c3c to restore RIT 2024-11-21T08:13:42,000 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732176852000 2024-11-21T08:13:42,000 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T08:13:42,000 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,002 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T08:13:42,012 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T08:13:42,012 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 021eaa39c912c55ccf9b758a5c92d1f8, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,012 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 021eaa39c912c55ccf9b758a5c92d1f8, server=6ed56fb6cbf7,35295,1732176812677 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,013 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd0a67dc199543d2675c547a7cb41702, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,013 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3773623806176f3d7f8efb3efe1b7937, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,014 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f38fd8b8defc002b7cb3de558717f27c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,014 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f43187c46588bff73d01ba7eb26a6f8, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,014 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd29f056dad8c6c37d0979434891505d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,014 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=829529fdccf9853d96113fa67713ffbc, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,014 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee7ef14c3df8915e550cfda528563489, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,015 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9eb0364d53b384c3e59ecc15348e6c3c, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,015 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd97d8f813bdfcba3626cc4950c7c1e1, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,015 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5fb158c8201cfbc5babdbfc33ee45ab1, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,015 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbfd5210cf46b2a246edeeebd244b662, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,015 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94c604fd0d435e28ac643b49d6cbee14, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,016 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d72691fc99dffac743de3d6bcb0f84ed, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,016 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99e7b3565f08f10e8c368072e9c21c09, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,016 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=477e65b3290a1e6e03e5eb59f7aa956d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,016 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5017ae3acb74fe8c7d0b9c6d9a2cc402, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,016 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d5b1f002057972e76beae8f19094030, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,017 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f55c025b8995c15117005602a775de64, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,017 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed3111e35326fc2c5455ae2475ab1b5, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,017 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec3a5c93577675c61d946ed41b85bdb5, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,017 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ef5fca53c8ff0d12fc112ab68731575, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=5 2024-11-21T08:13:42,017 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dfe569750c256f741c6d3e5ed072503, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,017 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8202a2e0e04a2c5c7f5f0e491d40e, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=021eaa39c912c55ccf9b758a5c92d1f8, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af43494f9c57dd49b6d504756610f96c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3589ece79012bcc3080c57b249902675, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,018 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T08:13:42,018 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732176882018 2024-11-21T08:13:42,018 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732176942018 2024-11-21T08:13:42,018 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 18 msec 2024-11-21T08:13:42,018 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T08:13:42,018 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,018 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T08:13:42,019 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T08:13:42,019 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T08:13:42,019 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T08:13:42,019 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T08:13:42,019 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T08:13:42,020 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T08:13:42,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T08:13:42,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:42,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:42,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:42,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:42,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,025 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,025 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,025 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:42,025 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,025 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:42,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T08:13:42,026 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T08:13:42,026 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, REOPEN/MOVE 2024-11-21T08:13:42,029 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true 2024-11-21T08:13:42,029 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,029 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,029 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T08:13:42,029 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,029 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,029 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,029 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:42,029 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:42,029 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:42,030 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN 2024-11-21T08:13:42,030 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN 2024-11-21T08:13:42,030 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN 2024-11-21T08:13:42,030 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN 2024-11-21T08:13:42,030 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN 2024-11-21T08:13:42,030 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,030 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd0a67dc199543d2675c547a7cb41702, server=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,030 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 021eaa39c912c55ccf9b758a5c92d1f8, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,030 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,030 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,030 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,030 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,030 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,030 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,030 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,031 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,031 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:42,031 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,031 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:42,031 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,031 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:42,031 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,031 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN 2024-11-21T08:13:42,031 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,031 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN 2024-11-21T08:13:42,031 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,031 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN 2024-11-21T08:13:42,031 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,031 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN 2024-11-21T08:13:42,031 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,031 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN 2024-11-21T08:13:42,031 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,032 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd0a67dc199543d2675c547a7cb41702, server=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,032 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 021eaa39c912c55ccf9b758a5c92d1f8, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,032 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-21T08:13:42,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=65 2024-11-21T08:13:42,033 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=hbase:meta, region=1588230740 2024-11-21T08:13:42,033 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ed56fb6cbf7,35295,1732176812677, state=OPEN 2024-11-21T08:13:42,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,046 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,046 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,046 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,047 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN to state=OFFLINE, location=null, table=test, region=021eaa39c912c55ccf9b758a5c92d1f8 to restore RIT 2024-11-21T08:13:42,047 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN to state=OFFLINE, location=null, table=test, region=cd29f056dad8c6c37d0979434891505d to restore RIT 2024-11-21T08:13:42,047 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN to state=OFFLINE, location=null, table=test, region=477e65b3290a1e6e03e5eb59f7aa956d to restore RIT 2024-11-21T08:13:42,047 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN to state=OFFLINE, location=null, table=test, region=dbfd5210cf46b2a246edeeebd244b662 to restore RIT 2024-11-21T08:13:42,047 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN to state=OFFLINE, location=null, table=test, region=fd0a67dc199543d2675c547a7cb41702 to restore RIT 2024-11-21T08:13:42,047 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN to state=OFFLINE, location=null, table=test, region=f55c025b8995c15117005602a775de64 to restore RIT 2024-11-21T08:13:42,047 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN to state=OFFLINE, location=null, table=test, region=3773623806176f3d7f8efb3efe1b7937 to restore RIT 2024-11-21T08:13:42,047 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN to state=OFFLINE, location=null, table=test, region=9eb0364d53b384c3e59ecc15348e6c3c to restore RIT 2024-11-21T08:13:42,047 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732176852047 2024-11-21T08:13:42,047 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T08:13:42,047 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,049 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T08:13:42,051 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T08:13:42,051 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd0a67dc199543d2675c547a7cb41702, server=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,051 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd0a67dc199543d2675c547a7cb41702, server=6ed56fb6cbf7,38845,1732176812866 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd0a67dc199543d2675c547a7cb41702, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3773623806176f3d7f8efb3efe1b7937, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f38fd8b8defc002b7cb3de558717f27c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f43187c46588bff73d01ba7eb26a6f8, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd29f056dad8c6c37d0979434891505d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=829529fdccf9853d96113fa67713ffbc, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee7ef14c3df8915e550cfda528563489, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9eb0364d53b384c3e59ecc15348e6c3c, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd97d8f813bdfcba3626cc4950c7c1e1, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5fb158c8201cfbc5babdbfc33ee45ab1, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbfd5210cf46b2a246edeeebd244b662, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94c604fd0d435e28ac643b49d6cbee14, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d72691fc99dffac743de3d6bcb0f84ed, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99e7b3565f08f10e8c368072e9c21c09, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=477e65b3290a1e6e03e5eb59f7aa956d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5017ae3acb74fe8c7d0b9c6d9a2cc402, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d5b1f002057972e76beae8f19094030, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f55c025b8995c15117005602a775de64, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed3111e35326fc2c5455ae2475ab1b5, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec3a5c93577675c61d946ed41b85bdb5, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ef5fca53c8ff0d12fc112ab68731575, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=5 2024-11-21T08:13:42,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dfe569750c256f741c6d3e5ed072503, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8202a2e0e04a2c5c7f5f0e491d40e, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=021eaa39c912c55ccf9b758a5c92d1f8, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af43494f9c57dd49b6d504756610f96c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3589ece79012bcc3080c57b249902675, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,057 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T08:13:42,057 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732176882057 2024-11-21T08:13:42,057 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732176942057 2024-11-21T08:13:42,057 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-21T08:13:42,057 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T08:13:42,057 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,057 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T08:13:42,057 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T08:13:42,057 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T08:13:42,057 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T08:13:42,057 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T08:13:42,058 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T08:13:42,058 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T08:13:42,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T08:13:42,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:42,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:42,062 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,062 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,062 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,062 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:42,062 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,062 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,062 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,062 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,062 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:42,062 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,062 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:42,062 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T08:13:42,063 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T08:13:42,064 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, REOPEN/MOVE 2024-11-21T08:13:42,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true 2024-11-21T08:13:42,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T08:13:42,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:42,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:42,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:42,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN 2024-11-21T08:13:42,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN 2024-11-21T08:13:42,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN 2024-11-21T08:13:42,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN 2024-11-21T08:13:42,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN 2024-11-21T08:13:42,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd0a67dc199543d2675c547a7cb41702, server=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,067 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,067 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,067 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:42,067 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,067 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:42,068 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,068 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:42,068 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,068 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN 2024-11-21T08:13:42,068 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,068 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN 2024-11-21T08:13:42,068 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,068 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN 2024-11-21T08:13:42,068 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,068 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN 2024-11-21T08:13:42,068 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,068 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN 2024-11-21T08:13:42,068 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,068 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd0a67dc199543d2675c547a7cb41702, server=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,068 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-21T08:13:42,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=65 2024-11-21T08:13:42,069 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=hbase:meta, region=1588230740 2024-11-21T08:13:42,069 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ed56fb6cbf7,35295,1732176812677, state=OPEN 2024-11-21T08:13:42,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,077 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,077 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,077 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,078 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN to state=OFFLINE, location=null, table=test, region=021eaa39c912c55ccf9b758a5c92d1f8 to restore RIT 2024-11-21T08:13:42,078 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN to state=OFFLINE, location=null, table=test, region=cd29f056dad8c6c37d0979434891505d to restore RIT 2024-11-21T08:13:42,078 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN to state=OFFLINE, location=null, table=test, region=477e65b3290a1e6e03e5eb59f7aa956d to restore RIT 2024-11-21T08:13:42,078 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN to state=OFFLINE, location=null, table=test, region=dbfd5210cf46b2a246edeeebd244b662 to restore RIT 2024-11-21T08:13:42,078 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN to state=OFFLINE, location=null, table=test, region=fd0a67dc199543d2675c547a7cb41702 to restore RIT 2024-11-21T08:13:42,078 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN to state=OFFLINE, location=null, table=test, region=f55c025b8995c15117005602a775de64 to restore RIT 2024-11-21T08:13:42,078 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN to state=OFFLINE, location=null, table=test, region=3773623806176f3d7f8efb3efe1b7937 to restore RIT 2024-11-21T08:13:42,078 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN to state=OFFLINE, location=null, table=test, region=9eb0364d53b384c3e59ecc15348e6c3c to restore RIT 2024-11-21T08:13:42,079 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732176852079 2024-11-21T08:13:42,079 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T08:13:42,079 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,080 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T08:13:42,081 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T08:13:42,081 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,081 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd0a67dc199543d2675c547a7cb41702, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3773623806176f3d7f8efb3efe1b7937, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f38fd8b8defc002b7cb3de558717f27c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f43187c46588bff73d01ba7eb26a6f8, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd29f056dad8c6c37d0979434891505d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=829529fdccf9853d96113fa67713ffbc, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee7ef14c3df8915e550cfda528563489, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,087 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9eb0364d53b384c3e59ecc15348e6c3c, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,087 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd97d8f813bdfcba3626cc4950c7c1e1, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,087 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5fb158c8201cfbc5babdbfc33ee45ab1, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,087 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbfd5210cf46b2a246edeeebd244b662, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,087 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94c604fd0d435e28ac643b49d6cbee14, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,087 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d72691fc99dffac743de3d6bcb0f84ed, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,087 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99e7b3565f08f10e8c368072e9c21c09, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,087 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=477e65b3290a1e6e03e5eb59f7aa956d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,087 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5017ae3acb74fe8c7d0b9c6d9a2cc402, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d5b1f002057972e76beae8f19094030, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f55c025b8995c15117005602a775de64, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed3111e35326fc2c5455ae2475ab1b5, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec3a5c93577675c61d946ed41b85bdb5, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ef5fca53c8ff0d12fc112ab68731575, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=5 2024-11-21T08:13:42,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dfe569750c256f741c6d3e5ed072503, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8202a2e0e04a2c5c7f5f0e491d40e, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=021eaa39c912c55ccf9b758a5c92d1f8, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af43494f9c57dd49b6d504756610f96c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3589ece79012bcc3080c57b249902675, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,089 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T08:13:42,089 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732176882089 2024-11-21T08:13:42,089 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732176942089 2024-11-21T08:13:42,089 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-21T08:13:42,089 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T08:13:42,089 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,089 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T08:13:42,089 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T08:13:42,089 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T08:13:42,089 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T08:13:42,089 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T08:13:42,090 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T08:13:42,090 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T08:13:42,092 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T08:13:42,092 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,092 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:42,092 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,092 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,092 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:42,092 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,092 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,092 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,092 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,092 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,092 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,092 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,093 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:42,093 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,093 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:42,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T08:13:42,094 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T08:13:42,094 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, REOPEN/MOVE 2024-11-21T08:13:42,096 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true 2024-11-21T08:13:42,096 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,096 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,096 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T08:13:42,096 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,096 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,096 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,096 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:42,096 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:42,096 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:42,096 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN 2024-11-21T08:13:42,096 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN 2024-11-21T08:13:42,096 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN 2024-11-21T08:13:42,096 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN 2024-11-21T08:13:42,096 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN 2024-11-21T08:13:42,096 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,096 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,096 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,096 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,096 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,096 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,097 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,097 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,097 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,097 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:42,097 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,097 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:42,097 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,097 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:42,097 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,097 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN 2024-11-21T08:13:42,097 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-21T08:13:42,097 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,097 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN 2024-11-21T08:13:42,097 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,097 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN 2024-11-21T08:13:42,097 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,097 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN 2024-11-21T08:13:42,097 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,097 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN 2024-11-21T08:13:42,097 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,098 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-21T08:13:42,098 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-21T08:13:42,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=65 2024-11-21T08:13:42,098 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=hbase:meta, region=1588230740 2024-11-21T08:13:42,099 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ed56fb6cbf7,35295,1732176812677, state=OPEN 2024-11-21T08:13:42,099 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test 2024-11-21T08:13:42,100 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test Metrics about Tables on a single HBase RegionServer 2024-11-21T08:13:42,101 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T08:13:42,101 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-21T08:13:42,101 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-21T08:13:42,101 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-21T08:13:42,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,109 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,109 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,109 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,109 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN to state=OFFLINE, location=null, table=test, region=021eaa39c912c55ccf9b758a5c92d1f8 to restore RIT 2024-11-21T08:13:42,110 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN to state=OFFLINE, location=null, table=test, region=cd29f056dad8c6c37d0979434891505d to restore RIT 2024-11-21T08:13:42,110 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN to state=OFFLINE, location=null, table=test, region=477e65b3290a1e6e03e5eb59f7aa956d to restore RIT 2024-11-21T08:13:42,110 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN to state=OFFLINE, location=null, table=test, region=dbfd5210cf46b2a246edeeebd244b662 to restore RIT 2024-11-21T08:13:42,110 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN to state=OFFLINE, location=null, table=test, region=fd0a67dc199543d2675c547a7cb41702 to restore RIT 2024-11-21T08:13:42,110 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN to state=OFFLINE, location=null, table=test, region=f55c025b8995c15117005602a775de64 to restore RIT 2024-11-21T08:13:42,110 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN to state=OFFLINE, location=null, table=test, region=3773623806176f3d7f8efb3efe1b7937 to restore RIT 2024-11-21T08:13:42,110 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN to state=OFFLINE, location=null, table=test, region=9eb0364d53b384c3e59ecc15348e6c3c to restore RIT 2024-11-21T08:13:42,110 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732176852110 2024-11-21T08:13:42,110 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T08:13:42,110 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,112 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T08:13:42,113 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T08:13:42,113 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN 2024-11-21T08:13:42,113 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,113 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd0a67dc199543d2675c547a7cb41702, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3773623806176f3d7f8efb3efe1b7937, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f38fd8b8defc002b7cb3de558717f27c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f43187c46588bff73d01ba7eb26a6f8, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd29f056dad8c6c37d0979434891505d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=829529fdccf9853d96113fa67713ffbc, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee7ef14c3df8915e550cfda528563489, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9eb0364d53b384c3e59ecc15348e6c3c, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd97d8f813bdfcba3626cc4950c7c1e1, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5fb158c8201cfbc5babdbfc33ee45ab1, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbfd5210cf46b2a246edeeebd244b662, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94c604fd0d435e28ac643b49d6cbee14, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d72691fc99dffac743de3d6bcb0f84ed, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99e7b3565f08f10e8c368072e9c21c09, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=477e65b3290a1e6e03e5eb59f7aa956d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5017ae3acb74fe8c7d0b9c6d9a2cc402, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d5b1f002057972e76beae8f19094030, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f55c025b8995c15117005602a775de64, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed3111e35326fc2c5455ae2475ab1b5, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec3a5c93577675c61d946ed41b85bdb5, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ef5fca53c8ff0d12fc112ab68731575, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=5 2024-11-21T08:13:42,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dfe569750c256f741c6d3e5ed072503, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8202a2e0e04a2c5c7f5f0e491d40e, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=021eaa39c912c55ccf9b758a5c92d1f8, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af43494f9c57dd49b6d504756610f96c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3589ece79012bcc3080c57b249902675, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,121 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T08:13:42,122 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732176882121 2024-11-21T08:13:42,122 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732176942122 2024-11-21T08:13:42,122 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-21T08:13:42,122 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T08:13:42,122 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,122 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T08:13:42,122 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T08:13:42,122 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T08:13:42,122 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T08:13:42,122 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T08:13:42,123 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T08:13:42,124 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T08:13:42,126 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T08:13:42,126 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,126 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:42,126 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,126 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,126 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,126 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,126 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,126 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,127 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,127 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,127 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,127 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:42,127 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,127 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:42,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T08:13:42,128 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T08:13:42,128 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, REOPEN/MOVE 2024-11-21T08:13:42,130 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true 2024-11-21T08:13:42,130 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,130 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,130 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T08:13:42,130 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,130 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,130 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,130 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:42,130 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:42,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:42,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN 2024-11-21T08:13:42,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN 2024-11-21T08:13:42,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN 2024-11-21T08:13:42,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN 2024-11-21T08:13:42,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN 2024-11-21T08:13:42,131 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,131 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,131 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,131 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,131 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,131 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,131 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,131 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,131 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:42,131 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,131 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:42,132 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,132 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:42,132 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,132 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN 2024-11-21T08:13:42,132 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,132 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN 2024-11-21T08:13:42,132 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,132 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN 2024-11-21T08:13:42,132 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,132 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN 2024-11-21T08:13:42,132 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,132 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN 2024-11-21T08:13:42,132 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-21T08:13:42,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=65 2024-11-21T08:13:42,134 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=hbase:meta, region=1588230740 2024-11-21T08:13:42,134 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ed56fb6cbf7,35295,1732176812677, state=OPEN 2024-11-21T08:13:42,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,140 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,140 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,140 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,141 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN to state=OFFLINE, location=null, table=test, region=021eaa39c912c55ccf9b758a5c92d1f8 to restore RIT 2024-11-21T08:13:42,141 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN to state=OFFLINE, location=null, table=test, region=cd29f056dad8c6c37d0979434891505d to restore RIT 2024-11-21T08:13:42,141 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN to state=OFFLINE, location=null, table=test, region=477e65b3290a1e6e03e5eb59f7aa956d to restore RIT 2024-11-21T08:13:42,141 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN to state=OFFLINE, location=null, table=test, region=dbfd5210cf46b2a246edeeebd244b662 to restore RIT 2024-11-21T08:13:42,142 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN to state=OFFLINE, location=null, table=test, region=fd0a67dc199543d2675c547a7cb41702 to restore RIT 2024-11-21T08:13:42,142 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN to state=OFFLINE, location=null, table=test, region=f55c025b8995c15117005602a775de64 to restore RIT 2024-11-21T08:13:42,142 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN to state=OFFLINE, location=null, table=test, region=3773623806176f3d7f8efb3efe1b7937 to restore RIT 2024-11-21T08:13:42,142 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9eb0364d53b384c3e59ecc15348e6c3c, ASSIGN to state=OFFLINE, location=null, table=test, region=9eb0364d53b384c3e59ecc15348e6c3c to restore RIT 2024-11-21T08:13:42,142 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732176852142 2024-11-21T08:13:42,142 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T08:13:42,142 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,144 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T08:13:42,145 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T08:13:42,145 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN 2024-11-21T08:13:42,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,146 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd0a67dc199543d2675c547a7cb41702, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3773623806176f3d7f8efb3efe1b7937, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f38fd8b8defc002b7cb3de558717f27c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f43187c46588bff73d01ba7eb26a6f8, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd29f056dad8c6c37d0979434891505d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=829529fdccf9853d96113fa67713ffbc, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee7ef14c3df8915e550cfda528563489, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9eb0364d53b384c3e59ecc15348e6c3c, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd97d8f813bdfcba3626cc4950c7c1e1, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5fb158c8201cfbc5babdbfc33ee45ab1, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbfd5210cf46b2a246edeeebd244b662, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94c604fd0d435e28ac643b49d6cbee14, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d72691fc99dffac743de3d6bcb0f84ed, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99e7b3565f08f10e8c368072e9c21c09, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=477e65b3290a1e6e03e5eb59f7aa956d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5017ae3acb74fe8c7d0b9c6d9a2cc402, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d5b1f002057972e76beae8f19094030, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f55c025b8995c15117005602a775de64, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed3111e35326fc2c5455ae2475ab1b5, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec3a5c93577675c61d946ed41b85bdb5, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ef5fca53c8ff0d12fc112ab68731575, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=5 2024-11-21T08:13:42,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dfe569750c256f741c6d3e5ed072503, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8202a2e0e04a2c5c7f5f0e491d40e, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=021eaa39c912c55ccf9b758a5c92d1f8, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af43494f9c57dd49b6d504756610f96c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3589ece79012bcc3080c57b249902675, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,150 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T08:13:42,151 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732176882150 2024-11-21T08:13:42,151 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732176942151 2024-11-21T08:13:42,151 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-21T08:13:42,151 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T08:13:42,151 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,151 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T08:13:42,151 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T08:13:42,151 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T08:13:42,151 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T08:13:42,151 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T08:13:42,152 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T08:13:42,152 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,152 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T08:13:42,152 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,154 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T08:13:42,154 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,154 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:42,154 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,154 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,154 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,154 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:42,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:42,155 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T08:13:42,155 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T08:13:42,156 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, REOPEN/MOVE 2024-11-21T08:13:42,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true 2024-11-21T08:13:42,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T08:13:42,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:42,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:42,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:42,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN 2024-11-21T08:13:42,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN 2024-11-21T08:13:42,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN 2024-11-21T08:13:42,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN 2024-11-21T08:13:42,157 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,158 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,158 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,158 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,158 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,158 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,158 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,158 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,158 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:42,158 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,158 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:42,158 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,158 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:42,158 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,158 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN 2024-11-21T08:13:42,158 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,158 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN 2024-11-21T08:13:42,158 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,158 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN 2024-11-21T08:13:42,158 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,158 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN 2024-11-21T08:13:42,158 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-21T08:13:42,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=65 2024-11-21T08:13:42,159 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=hbase:meta, region=1588230740 2024-11-21T08:13:42,159 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ed56fb6cbf7,35295,1732176812677, state=OPEN 2024-11-21T08:13:42,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,172 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,172 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,172 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,173 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN to state=OFFLINE, location=null, table=test, region=021eaa39c912c55ccf9b758a5c92d1f8 to restore RIT 2024-11-21T08:13:42,173 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN to state=OFFLINE, location=null, table=test, region=cd29f056dad8c6c37d0979434891505d to restore RIT 2024-11-21T08:13:42,173 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN to state=OFFLINE, location=null, table=test, region=477e65b3290a1e6e03e5eb59f7aa956d to restore RIT 2024-11-21T08:13:42,173 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN to state=OFFLINE, location=null, table=test, region=dbfd5210cf46b2a246edeeebd244b662 to restore RIT 2024-11-21T08:13:42,173 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN to state=OFFLINE, location=null, table=test, region=fd0a67dc199543d2675c547a7cb41702 to restore RIT 2024-11-21T08:13:42,173 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN to state=OFFLINE, location=null, table=test, region=f55c025b8995c15117005602a775de64 to restore RIT 2024-11-21T08:13:42,173 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3773623806176f3d7f8efb3efe1b7937, ASSIGN to state=OFFLINE, location=null, table=test, region=3773623806176f3d7f8efb3efe1b7937 to restore RIT 2024-11-21T08:13:42,174 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732176852174 2024-11-21T08:13:42,174 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T08:13:42,174 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,175 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T08:13:42,176 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T08:13:42,176 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN 2024-11-21T08:13:42,176 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,176 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd0a67dc199543d2675c547a7cb41702, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3773623806176f3d7f8efb3efe1b7937, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f38fd8b8defc002b7cb3de558717f27c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f43187c46588bff73d01ba7eb26a6f8, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd29f056dad8c6c37d0979434891505d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=829529fdccf9853d96113fa67713ffbc, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee7ef14c3df8915e550cfda528563489, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,182 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9eb0364d53b384c3e59ecc15348e6c3c, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,182 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd97d8f813bdfcba3626cc4950c7c1e1, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,182 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5fb158c8201cfbc5babdbfc33ee45ab1, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,182 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbfd5210cf46b2a246edeeebd244b662, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,182 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94c604fd0d435e28ac643b49d6cbee14, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,182 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d72691fc99dffac743de3d6bcb0f84ed, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99e7b3565f08f10e8c368072e9c21c09, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=477e65b3290a1e6e03e5eb59f7aa956d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5017ae3acb74fe8c7d0b9c6d9a2cc402, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d5b1f002057972e76beae8f19094030, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f55c025b8995c15117005602a775de64, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed3111e35326fc2c5455ae2475ab1b5, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec3a5c93577675c61d946ed41b85bdb5, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ef5fca53c8ff0d12fc112ab68731575, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=5 2024-11-21T08:13:42,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dfe569750c256f741c6d3e5ed072503, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8202a2e0e04a2c5c7f5f0e491d40e, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=021eaa39c912c55ccf9b758a5c92d1f8, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af43494f9c57dd49b6d504756610f96c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3589ece79012bcc3080c57b249902675, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,185 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T08:13:42,185 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732176882185 2024-11-21T08:13:42,185 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732176942185 2024-11-21T08:13:42,185 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-21T08:13:42,185 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T08:13:42,185 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,185 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T08:13:42,185 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T08:13:42,185 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T08:13:42,185 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T08:13:42,186 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T08:13:42,186 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T08:13:42,187 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T08:13:42,189 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T08:13:42,189 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,189 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:42,189 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,189 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,189 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:42,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:42,190 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T08:13:42,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T08:13:42,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, REOPEN/MOVE 2024-11-21T08:13:42,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true 2024-11-21T08:13:42,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T08:13:42,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:42,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:42,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:42,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN 2024-11-21T08:13:42,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN 2024-11-21T08:13:42,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN 2024-11-21T08:13:42,193 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,193 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,194 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,194 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,194 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,194 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,194 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,194 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,194 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:42,194 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,194 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:42,194 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,194 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:42,194 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,194 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN 2024-11-21T08:13:42,194 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,194 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN 2024-11-21T08:13:42,195 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,195 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN 2024-11-21T08:13:42,195 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-21T08:13:42,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=65 2024-11-21T08:13:42,196 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=hbase:meta, region=1588230740 2024-11-21T08:13:42,196 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ed56fb6cbf7,35295,1732176812677, state=OPEN 2024-11-21T08:13:42,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,203 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,203 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,204 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,204 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN to state=OFFLINE, location=null, table=test, region=021eaa39c912c55ccf9b758a5c92d1f8 to restore RIT 2024-11-21T08:13:42,205 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN to state=OFFLINE, location=null, table=test, region=cd29f056dad8c6c37d0979434891505d to restore RIT 2024-11-21T08:13:42,205 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN to state=OFFLINE, location=null, table=test, region=477e65b3290a1e6e03e5eb59f7aa956d to restore RIT 2024-11-21T08:13:42,205 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN to state=OFFLINE, location=null, table=test, region=dbfd5210cf46b2a246edeeebd244b662 to restore RIT 2024-11-21T08:13:42,205 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN to state=OFFLINE, location=null, table=test, region=fd0a67dc199543d2675c547a7cb41702 to restore RIT 2024-11-21T08:13:42,205 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f55c025b8995c15117005602a775de64, ASSIGN to state=OFFLINE, location=null, table=test, region=f55c025b8995c15117005602a775de64 to restore RIT 2024-11-21T08:13:42,205 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732176852205 2024-11-21T08:13:42,205 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T08:13:42,205 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,206 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T08:13:42,208 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T08:13:42,208 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN 2024-11-21T08:13:42,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,209 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd0a67dc199543d2675c547a7cb41702, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3773623806176f3d7f8efb3efe1b7937, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f38fd8b8defc002b7cb3de558717f27c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f43187c46588bff73d01ba7eb26a6f8, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd29f056dad8c6c37d0979434891505d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=829529fdccf9853d96113fa67713ffbc, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee7ef14c3df8915e550cfda528563489, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9eb0364d53b384c3e59ecc15348e6c3c, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd97d8f813bdfcba3626cc4950c7c1e1, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5fb158c8201cfbc5babdbfc33ee45ab1, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbfd5210cf46b2a246edeeebd244b662, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94c604fd0d435e28ac643b49d6cbee14, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d72691fc99dffac743de3d6bcb0f84ed, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99e7b3565f08f10e8c368072e9c21c09, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=477e65b3290a1e6e03e5eb59f7aa956d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5017ae3acb74fe8c7d0b9c6d9a2cc402, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d5b1f002057972e76beae8f19094030, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f55c025b8995c15117005602a775de64, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed3111e35326fc2c5455ae2475ab1b5, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec3a5c93577675c61d946ed41b85bdb5, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ef5fca53c8ff0d12fc112ab68731575, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=5 2024-11-21T08:13:42,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dfe569750c256f741c6d3e5ed072503, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8202a2e0e04a2c5c7f5f0e491d40e, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=021eaa39c912c55ccf9b758a5c92d1f8, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af43494f9c57dd49b6d504756610f96c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3589ece79012bcc3080c57b249902675, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,213 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T08:13:42,213 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732176882213 2024-11-21T08:13:42,213 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732176942213 2024-11-21T08:13:42,213 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-21T08:13:42,213 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T08:13:42,213 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,213 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T08:13:42,213 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T08:13:42,213 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T08:13:42,213 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T08:13:42,213 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T08:13:42,214 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T08:13:42,214 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T08:13:42,216 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T08:13:42,216 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,216 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:42,216 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,216 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,216 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,216 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,216 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,216 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,216 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:42,216 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,216 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:42,216 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T08:13:42,217 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T08:13:42,217 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, REOPEN/MOVE 2024-11-21T08:13:42,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true 2024-11-21T08:13:42,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T08:13:42,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:42,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:42,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:42,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN 2024-11-21T08:13:42,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN 2024-11-21T08:13:42,218 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,218 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,218 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,218 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,218 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,218 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,218 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,218 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,219 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:42,219 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,219 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:42,219 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,219 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:42,219 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,219 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN 2024-11-21T08:13:42,219 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,219 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN 2024-11-21T08:13:42,219 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-21T08:13:42,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=65 2024-11-21T08:13:42,220 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=hbase:meta, region=1588230740 2024-11-21T08:13:42,220 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ed56fb6cbf7,35295,1732176812677, state=OPEN 2024-11-21T08:13:42,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,225 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,225 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,225 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,226 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN to state=OFFLINE, location=null, table=test, region=021eaa39c912c55ccf9b758a5c92d1f8 to restore RIT 2024-11-21T08:13:42,226 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN to state=OFFLINE, location=null, table=test, region=cd29f056dad8c6c37d0979434891505d to restore RIT 2024-11-21T08:13:42,226 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN to state=OFFLINE, location=null, table=test, region=477e65b3290a1e6e03e5eb59f7aa956d to restore RIT 2024-11-21T08:13:42,226 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN to state=OFFLINE, location=null, table=test, region=dbfd5210cf46b2a246edeeebd244b662 to restore RIT 2024-11-21T08:13:42,226 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fd0a67dc199543d2675c547a7cb41702, ASSIGN to state=OFFLINE, location=null, table=test, region=fd0a67dc199543d2675c547a7cb41702 to restore RIT 2024-11-21T08:13:42,227 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732176852227 2024-11-21T08:13:42,227 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T08:13:42,227 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,228 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T08:13:42,229 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T08:13:42,229 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN 2024-11-21T08:13:42,229 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,229 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,233 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd0a67dc199543d2675c547a7cb41702, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,233 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3773623806176f3d7f8efb3efe1b7937, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,233 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f38fd8b8defc002b7cb3de558717f27c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,233 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f43187c46588bff73d01ba7eb26a6f8, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,233 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd29f056dad8c6c37d0979434891505d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=829529fdccf9853d96113fa67713ffbc, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee7ef14c3df8915e550cfda528563489, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9eb0364d53b384c3e59ecc15348e6c3c, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd97d8f813bdfcba3626cc4950c7c1e1, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5fb158c8201cfbc5babdbfc33ee45ab1, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbfd5210cf46b2a246edeeebd244b662, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94c604fd0d435e28ac643b49d6cbee14, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d72691fc99dffac743de3d6bcb0f84ed, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99e7b3565f08f10e8c368072e9c21c09, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=477e65b3290a1e6e03e5eb59f7aa956d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5017ae3acb74fe8c7d0b9c6d9a2cc402, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d5b1f002057972e76beae8f19094030, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f55c025b8995c15117005602a775de64, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed3111e35326fc2c5455ae2475ab1b5, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec3a5c93577675c61d946ed41b85bdb5, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ef5fca53c8ff0d12fc112ab68731575, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=5 2024-11-21T08:13:42,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dfe569750c256f741c6d3e5ed072503, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8202a2e0e04a2c5c7f5f0e491d40e, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=021eaa39c912c55ccf9b758a5c92d1f8, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af43494f9c57dd49b6d504756610f96c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3589ece79012bcc3080c57b249902675, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,236 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T08:13:42,236 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732176882236 2024-11-21T08:13:42,236 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732176942236 2024-11-21T08:13:42,236 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-21T08:13:42,236 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T08:13:42,236 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,236 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T08:13:42,236 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T08:13:42,236 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T08:13:42,236 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T08:13:42,236 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T08:13:42,237 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T08:13:42,237 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T08:13:42,239 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T08:13:42,239 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,239 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:42,239 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,240 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,240 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,240 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,240 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,240 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:42,240 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,240 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:42,240 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T08:13:42,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T08:13:42,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, REOPEN/MOVE 2024-11-21T08:13:42,242 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true 2024-11-21T08:13:42,242 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,242 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,242 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T08:13:42,242 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,242 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,242 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,242 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:42,242 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:42,242 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:42,242 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN 2024-11-21T08:13:42,242 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,242 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,242 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,243 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,243 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,243 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,243 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,243 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,243 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:42,243 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,243 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:42,243 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,243 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:42,243 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,243 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN 2024-11-21T08:13:42,243 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-21T08:13:42,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=65 2024-11-21T08:13:42,244 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=hbase:meta, region=1588230740 2024-11-21T08:13:42,244 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ed56fb6cbf7,35295,1732176812677, state=OPEN 2024-11-21T08:13:42,255 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,255 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,312 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,312 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,312 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,313 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN to state=OFFLINE, location=null, table=test, region=021eaa39c912c55ccf9b758a5c92d1f8 to restore RIT 2024-11-21T08:13:42,313 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN to state=OFFLINE, location=null, table=test, region=cd29f056dad8c6c37d0979434891505d to restore RIT 2024-11-21T08:13:42,313 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN to state=OFFLINE, location=null, table=test, region=477e65b3290a1e6e03e5eb59f7aa956d to restore RIT 2024-11-21T08:13:42,313 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dbfd5210cf46b2a246edeeebd244b662, ASSIGN to state=OFFLINE, location=null, table=test, region=dbfd5210cf46b2a246edeeebd244b662 to restore RIT 2024-11-21T08:13:42,314 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732176852314 2024-11-21T08:13:42,314 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T08:13:42,314 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,316 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T08:13:42,317 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T08:13:42,317 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:42,317 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,317 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd0a67dc199543d2675c547a7cb41702, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3773623806176f3d7f8efb3efe1b7937, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f38fd8b8defc002b7cb3de558717f27c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f43187c46588bff73d01ba7eb26a6f8, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd29f056dad8c6c37d0979434891505d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=829529fdccf9853d96113fa67713ffbc, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee7ef14c3df8915e550cfda528563489, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9eb0364d53b384c3e59ecc15348e6c3c, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd97d8f813bdfcba3626cc4950c7c1e1, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5fb158c8201cfbc5babdbfc33ee45ab1, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbfd5210cf46b2a246edeeebd244b662, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94c604fd0d435e28ac643b49d6cbee14, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d72691fc99dffac743de3d6bcb0f84ed, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99e7b3565f08f10e8c368072e9c21c09, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=477e65b3290a1e6e03e5eb59f7aa956d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5017ae3acb74fe8c7d0b9c6d9a2cc402, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d5b1f002057972e76beae8f19094030, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f55c025b8995c15117005602a775de64, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed3111e35326fc2c5455ae2475ab1b5, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec3a5c93577675c61d946ed41b85bdb5, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ef5fca53c8ff0d12fc112ab68731575, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=5 2024-11-21T08:13:42,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dfe569750c256f741c6d3e5ed072503, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8202a2e0e04a2c5c7f5f0e491d40e, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=021eaa39c912c55ccf9b758a5c92d1f8, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af43494f9c57dd49b6d504756610f96c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3589ece79012bcc3080c57b249902675, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,322 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T08:13:42,322 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732176882322 2024-11-21T08:13:42,322 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732176942322 2024-11-21T08:13:42,322 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-21T08:13:42,322 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T08:13:42,322 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,322 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T08:13:42,322 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T08:13:42,323 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T08:13:42,323 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T08:13:42,323 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T08:13:42,323 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T08:13:42,324 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T08:13:42,325 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T08:13:42,325 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,325 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:42,325 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,325 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,325 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,325 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,325 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:42,325 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,325 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:42,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T08:13:42,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T08:13:42,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, REOPEN/MOVE 2024-11-21T08:13:42,327 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true 2024-11-21T08:13:42,327 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,327 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,327 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T08:13:42,327 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,327 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,327 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,328 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:42,328 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:42,328 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:42,328 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,328 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,328 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,328 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,328 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,328 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,328 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,328 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,328 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:42,328 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,328 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:42,328 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,328 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN 2024-11-21T08:13:42,328 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-21T08:13:42,329 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=65 2024-11-21T08:13:42,329 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=hbase:meta, region=1588230740 2024-11-21T08:13:42,329 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ed56fb6cbf7,35295,1732176812677, state=OPEN 2024-11-21T08:13:42,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,361 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,361 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,361 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,362 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN to state=OFFLINE, location=null, table=test, region=021eaa39c912c55ccf9b758a5c92d1f8 to restore RIT 2024-11-21T08:13:42,362 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN to state=OFFLINE, location=null, table=test, region=cd29f056dad8c6c37d0979434891505d to restore RIT 2024-11-21T08:13:42,362 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=477e65b3290a1e6e03e5eb59f7aa956d, ASSIGN to state=OFFLINE, location=null, table=test, region=477e65b3290a1e6e03e5eb59f7aa956d to restore RIT 2024-11-21T08:13:42,363 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732176852363 2024-11-21T08:13:42,363 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T08:13:42,363 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,364 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T08:13:42,365 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T08:13:42,365 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:42,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,366 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,369 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd0a67dc199543d2675c547a7cb41702, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,369 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3773623806176f3d7f8efb3efe1b7937, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,370 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f38fd8b8defc002b7cb3de558717f27c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,370 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f43187c46588bff73d01ba7eb26a6f8, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,370 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd29f056dad8c6c37d0979434891505d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,370 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=829529fdccf9853d96113fa67713ffbc, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,370 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee7ef14c3df8915e550cfda528563489, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,370 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9eb0364d53b384c3e59ecc15348e6c3c, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,370 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd97d8f813bdfcba3626cc4950c7c1e1, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,370 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5fb158c8201cfbc5babdbfc33ee45ab1, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,371 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbfd5210cf46b2a246edeeebd244b662, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,371 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94c604fd0d435e28ac643b49d6cbee14, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,371 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d72691fc99dffac743de3d6bcb0f84ed, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,371 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99e7b3565f08f10e8c368072e9c21c09, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,371 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=477e65b3290a1e6e03e5eb59f7aa956d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,371 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5017ae3acb74fe8c7d0b9c6d9a2cc402, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,371 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d5b1f002057972e76beae8f19094030, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,371 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f55c025b8995c15117005602a775de64, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,371 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed3111e35326fc2c5455ae2475ab1b5, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,372 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec3a5c93577675c61d946ed41b85bdb5, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,372 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ef5fca53c8ff0d12fc112ab68731575, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=5 2024-11-21T08:13:42,372 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dfe569750c256f741c6d3e5ed072503, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,372 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8202a2e0e04a2c5c7f5f0e491d40e, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,372 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=021eaa39c912c55ccf9b758a5c92d1f8, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,372 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af43494f9c57dd49b6d504756610f96c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,372 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3589ece79012bcc3080c57b249902675, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,372 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T08:13:42,372 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732176882372 2024-11-21T08:13:42,373 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732176942372 2024-11-21T08:13:42,373 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-21T08:13:42,373 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T08:13:42,373 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,373 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T08:13:42,373 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T08:13:42,373 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T08:13:42,373 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T08:13:42,373 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T08:13:42,373 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T08:13:42,374 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T08:13:42,376 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T08:13:42,376 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,376 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:42,376 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,376 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,376 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,377 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:42,377 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,377 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:42,377 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T08:13:42,377 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T08:13:42,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, REOPEN/MOVE 2024-11-21T08:13:42,379 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true 2024-11-21T08:13:42,379 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,379 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,379 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T08:13:42,379 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,379 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,379 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,379 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:42,379 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:42,379 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,379 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,379 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,379 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,379 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,379 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,379 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,379 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,379 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:42,380 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,380 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN 2024-11-21T08:13:42,380 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-21T08:13:42,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=65 2024-11-21T08:13:42,381 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=hbase:meta, region=1588230740 2024-11-21T08:13:42,381 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ed56fb6cbf7,35295,1732176812677, state=OPEN 2024-11-21T08:13:42,414 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,415 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,451 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,451 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,451 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,452 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN to state=OFFLINE, location=null, table=test, region=021eaa39c912c55ccf9b758a5c92d1f8 to restore RIT 2024-11-21T08:13:42,453 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd29f056dad8c6c37d0979434891505d, ASSIGN to state=OFFLINE, location=null, table=test, region=cd29f056dad8c6c37d0979434891505d to restore RIT 2024-11-21T08:13:42,455 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732176852455 2024-11-21T08:13:42,455 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T08:13:42,455 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,457 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T08:13:42,458 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T08:13:42,458 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:42,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,458 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd0a67dc199543d2675c547a7cb41702, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3773623806176f3d7f8efb3efe1b7937, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f38fd8b8defc002b7cb3de558717f27c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f43187c46588bff73d01ba7eb26a6f8, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd29f056dad8c6c37d0979434891505d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=829529fdccf9853d96113fa67713ffbc, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee7ef14c3df8915e550cfda528563489, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9eb0364d53b384c3e59ecc15348e6c3c, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd97d8f813bdfcba3626cc4950c7c1e1, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5fb158c8201cfbc5babdbfc33ee45ab1, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbfd5210cf46b2a246edeeebd244b662, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94c604fd0d435e28ac643b49d6cbee14, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d72691fc99dffac743de3d6bcb0f84ed, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99e7b3565f08f10e8c368072e9c21c09, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=477e65b3290a1e6e03e5eb59f7aa956d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5017ae3acb74fe8c7d0b9c6d9a2cc402, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d5b1f002057972e76beae8f19094030, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f55c025b8995c15117005602a775de64, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed3111e35326fc2c5455ae2475ab1b5, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec3a5c93577675c61d946ed41b85bdb5, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ef5fca53c8ff0d12fc112ab68731575, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=5 2024-11-21T08:13:42,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dfe569750c256f741c6d3e5ed072503, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8202a2e0e04a2c5c7f5f0e491d40e, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=021eaa39c912c55ccf9b758a5c92d1f8, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af43494f9c57dd49b6d504756610f96c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3589ece79012bcc3080c57b249902675, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,464 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T08:13:42,464 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732176882464 2024-11-21T08:13:42,464 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732176942464 2024-11-21T08:13:42,464 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-21T08:13:42,464 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T08:13:42,464 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,464 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T08:13:42,464 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T08:13:42,465 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T08:13:42,465 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T08:13:42,465 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T08:13:42,465 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T08:13:42,466 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T08:13:42,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T08:13:42,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:42,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:42,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:42,469 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T08:13:42,470 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T08:13:42,470 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, REOPEN/MOVE 2024-11-21T08:13:42,471 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true 2024-11-21T08:13:42,471 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,471 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,471 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T08:13:42,472 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,472 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,472 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,472 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:42,472 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,472 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,472 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,472 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,472 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,472 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,472 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,472 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,472 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN 2024-11-21T08:13:42,472 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-21T08:13:42,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=65 2024-11-21T08:13:42,473 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=hbase:meta, region=1588230740 2024-11-21T08:13:42,473 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ed56fb6cbf7,35295,1732176812677, state=OPEN 2024-11-21T08:13:42,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,488 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,488 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,488 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,488 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=021eaa39c912c55ccf9b758a5c92d1f8, ASSIGN to state=OFFLINE, location=null, table=test, region=021eaa39c912c55ccf9b758a5c92d1f8 to restore RIT 2024-11-21T08:13:42,489 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732176852489 2024-11-21T08:13:42,489 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T08:13:42,489 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,490 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T08:13:42,491 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T08:13:42,491 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,491 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd0a67dc199543d2675c547a7cb41702, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3773623806176f3d7f8efb3efe1b7937, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f38fd8b8defc002b7cb3de558717f27c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f43187c46588bff73d01ba7eb26a6f8, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd29f056dad8c6c37d0979434891505d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=829529fdccf9853d96113fa67713ffbc, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee7ef14c3df8915e550cfda528563489, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,496 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9eb0364d53b384c3e59ecc15348e6c3c, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,496 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd97d8f813bdfcba3626cc4950c7c1e1, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,496 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5fb158c8201cfbc5babdbfc33ee45ab1, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,496 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbfd5210cf46b2a246edeeebd244b662, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,496 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94c604fd0d435e28ac643b49d6cbee14, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,496 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d72691fc99dffac743de3d6bcb0f84ed, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,496 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99e7b3565f08f10e8c368072e9c21c09, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,496 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=477e65b3290a1e6e03e5eb59f7aa956d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5017ae3acb74fe8c7d0b9c6d9a2cc402, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d5b1f002057972e76beae8f19094030, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f55c025b8995c15117005602a775de64, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed3111e35326fc2c5455ae2475ab1b5, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec3a5c93577675c61d946ed41b85bdb5, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ef5fca53c8ff0d12fc112ab68731575, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=5 2024-11-21T08:13:42,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dfe569750c256f741c6d3e5ed072503, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8202a2e0e04a2c5c7f5f0e491d40e, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=021eaa39c912c55ccf9b758a5c92d1f8, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af43494f9c57dd49b6d504756610f96c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3589ece79012bcc3080c57b249902675, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,498 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T08:13:42,498 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732176882498 2024-11-21T08:13:42,498 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732176942498 2024-11-21T08:13:42,498 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-21T08:13:42,498 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T08:13:42,498 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,498 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T08:13:42,498 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T08:13:42,499 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T08:13:42,499 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T08:13:42,499 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T08:13:42,499 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T08:13:42,500 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T08:13:42,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T08:13:42,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:42,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:42,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:42,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T08:13:42,503 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T08:13:42,503 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, REOPEN/MOVE 2024-11-21T08:13:42,504 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true 2024-11-21T08:13:42,504 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,504 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,504 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T08:13:42,504 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,504 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,504 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,504 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,504 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,504 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,505 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,505 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,505 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,505 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,505 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-21T08:13:42,506 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=65 2024-11-21T08:13:42,506 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=hbase:meta, region=1588230740 2024-11-21T08:13:42,506 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ed56fb6cbf7,35295,1732176812677, state=OPEN 2024-11-21T08:13:42,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,519 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,519 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,519 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,520 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732176852520 2024-11-21T08:13:42,520 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T08:13:42,520 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,521 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T08:13:42,523 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T08:13:42,523 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,523 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd0a67dc199543d2675c547a7cb41702, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3773623806176f3d7f8efb3efe1b7937, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f38fd8b8defc002b7cb3de558717f27c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f43187c46588bff73d01ba7eb26a6f8, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd29f056dad8c6c37d0979434891505d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=829529fdccf9853d96113fa67713ffbc, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee7ef14c3df8915e550cfda528563489, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9eb0364d53b384c3e59ecc15348e6c3c, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd97d8f813bdfcba3626cc4950c7c1e1, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5fb158c8201cfbc5babdbfc33ee45ab1, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbfd5210cf46b2a246edeeebd244b662, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94c604fd0d435e28ac643b49d6cbee14, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d72691fc99dffac743de3d6bcb0f84ed, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99e7b3565f08f10e8c368072e9c21c09, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=477e65b3290a1e6e03e5eb59f7aa956d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5017ae3acb74fe8c7d0b9c6d9a2cc402, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d5b1f002057972e76beae8f19094030, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f55c025b8995c15117005602a775de64, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed3111e35326fc2c5455ae2475ab1b5, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec3a5c93577675c61d946ed41b85bdb5, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ef5fca53c8ff0d12fc112ab68731575, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=5 2024-11-21T08:13:42,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dfe569750c256f741c6d3e5ed072503, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8202a2e0e04a2c5c7f5f0e491d40e, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=021eaa39c912c55ccf9b758a5c92d1f8, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af43494f9c57dd49b6d504756610f96c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3589ece79012bcc3080c57b249902675, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,528 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T08:13:42,528 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732176882528 2024-11-21T08:13:42,528 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732176942528 2024-11-21T08:13:42,528 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-21T08:13:42,528 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T08:13:42,528 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,528 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T08:13:42,528 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T08:13:42,529 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T08:13:42,529 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T08:13:42,529 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T08:13:42,529 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T08:13:42,530 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T08:13:42,531 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T08:13:42,531 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,531 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,531 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:42,531 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,531 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:42,531 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T08:13:42,532 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T08:13:42,532 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, REOPEN/MOVE 2024-11-21T08:13:42,532 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true 2024-11-21T08:13:42,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T08:13:42,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,533 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,533 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,533 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,533 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,533 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,533 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.1732176815010, worker=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,533 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-21T08:13:42,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=65 2024-11-21T08:13:42,534 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=hbase:meta, region=1588230740 2024-11-21T08:13:42,534 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ed56fb6cbf7,35295,1732176812677, state=OPEN 2024-11-21T08:13:42,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,540 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,540 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,540 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,542 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732176852542 2024-11-21T08:13:42,543 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T08:13:42,543 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,544 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T08:13:42,547 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T08:13:42,547 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,547 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,548 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd0a67dc199543d2675c547a7cb41702, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,548 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3773623806176f3d7f8efb3efe1b7937, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,548 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f38fd8b8defc002b7cb3de558717f27c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,548 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f43187c46588bff73d01ba7eb26a6f8, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,548 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd29f056dad8c6c37d0979434891505d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=829529fdccf9853d96113fa67713ffbc, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee7ef14c3df8915e550cfda528563489, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9eb0364d53b384c3e59ecc15348e6c3c, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd97d8f813bdfcba3626cc4950c7c1e1, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5fb158c8201cfbc5babdbfc33ee45ab1, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbfd5210cf46b2a246edeeebd244b662, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94c604fd0d435e28ac643b49d6cbee14, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d72691fc99dffac743de3d6bcb0f84ed, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99e7b3565f08f10e8c368072e9c21c09, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=477e65b3290a1e6e03e5eb59f7aa956d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5017ae3acb74fe8c7d0b9c6d9a2cc402, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d5b1f002057972e76beae8f19094030, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f55c025b8995c15117005602a775de64, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed3111e35326fc2c5455ae2475ab1b5, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec3a5c93577675c61d946ed41b85bdb5, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ef5fca53c8ff0d12fc112ab68731575, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=5 2024-11-21T08:13:42,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dfe569750c256f741c6d3e5ed072503, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,551 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8202a2e0e04a2c5c7f5f0e491d40e, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,551 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=021eaa39c912c55ccf9b758a5c92d1f8, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,551 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af43494f9c57dd49b6d504756610f96c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,551 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3589ece79012bcc3080c57b249902675, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,551 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T08:13:42,551 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732176882551 2024-11-21T08:13:42,551 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732176942551 2024-11-21T08:13:42,551 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-21T08:13:42,551 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T08:13:42,551 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,551 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T08:13:42,551 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T08:13:42,552 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T08:13:42,552 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T08:13:42,552 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T08:13:42,552 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T08:13:42,553 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T08:13:42,553 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,554 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,555 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T08:13:42,555 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,555 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:42,555 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,555 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T08:13:42,556 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T08:13:42,556 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T08:13:42,556 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, REOPEN/MOVE 2024-11-21T08:13:42,557 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true 2024-11-21T08:13:42,557 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,557 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,557 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T08:13:42,557 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,557 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,557 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,557 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,558 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,558 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ed56fb6cbf7,35295,1732176812677 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,558 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-21T08:13:42,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=65 2024-11-21T08:13:42,559 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=hbase:meta, region=1588230740 2024-11-21T08:13:42,559 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ed56fb6cbf7,35295,1732176812677, state=OPEN 2024-11-21T08:13:42,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,572 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,572 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,573 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,574 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732176852574 2024-11-21T08:13:42,574 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T08:13:42,574 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,575 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T08:13:42,577 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T08:13:42,577 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T08:13:42,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,577 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd0a67dc199543d2675c547a7cb41702, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3773623806176f3d7f8efb3efe1b7937, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f38fd8b8defc002b7cb3de558717f27c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f43187c46588bff73d01ba7eb26a6f8, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd29f056dad8c6c37d0979434891505d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=829529fdccf9853d96113fa67713ffbc, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee7ef14c3df8915e550cfda528563489, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9eb0364d53b384c3e59ecc15348e6c3c, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd97d8f813bdfcba3626cc4950c7c1e1, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5fb158c8201cfbc5babdbfc33ee45ab1, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbfd5210cf46b2a246edeeebd244b662, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94c604fd0d435e28ac643b49d6cbee14, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d72691fc99dffac743de3d6bcb0f84ed, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99e7b3565f08f10e8c368072e9c21c09, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=477e65b3290a1e6e03e5eb59f7aa956d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5017ae3acb74fe8c7d0b9c6d9a2cc402, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d5b1f002057972e76beae8f19094030, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f55c025b8995c15117005602a775de64, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,581 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed3111e35326fc2c5455ae2475ab1b5, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,581 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec3a5c93577675c61d946ed41b85bdb5, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,581 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ef5fca53c8ff0d12fc112ab68731575, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=5 2024-11-21T08:13:42,581 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dfe569750c256f741c6d3e5ed072503, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,581 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8202a2e0e04a2c5c7f5f0e491d40e, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,581 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=021eaa39c912c55ccf9b758a5c92d1f8, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,581 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af43494f9c57dd49b6d504756610f96c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,581 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3589ece79012bcc3080c57b249902675, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,581 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T08:13:42,581 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732176882581 2024-11-21T08:13:42,581 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732176942581 2024-11-21T08:13:42,581 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-21T08:13:42,582 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T08:13:42,582 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,582 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T08:13:42,582 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T08:13:42,582 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T08:13:42,582 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T08:13:42,582 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T08:13:42,582 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T08:13:42,583 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T08:13:42,585 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T08:13:42,585 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,585 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:42,585 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T08:13:42,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T08:13:42,586 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T08:13:42,586 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, REOPEN/MOVE 2024-11-21T08:13:42,587 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true 2024-11-21T08:13:42,587 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,587 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,587 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T08:13:42,587 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,587 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,587 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,587 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,587 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-21T08:13:42,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=65 2024-11-21T08:13:42,588 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=hbase:meta, region=1588230740 2024-11-21T08:13:42,588 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ed56fb6cbf7,35295,1732176812677, state=OPEN 2024-11-21T08:13:42,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,593 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,593 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,593 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,594 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732176852594 2024-11-21T08:13:42,594 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T08:13:42,594 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,596 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T08:13:42,600 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T08:13:42,600 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,600 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd0a67dc199543d2675c547a7cb41702, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,600 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3773623806176f3d7f8efb3efe1b7937, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,600 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f38fd8b8defc002b7cb3de558717f27c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,600 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,600 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f43187c46588bff73d01ba7eb26a6f8, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,600 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd29f056dad8c6c37d0979434891505d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,600 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=829529fdccf9853d96113fa67713ffbc, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,600 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee7ef14c3df8915e550cfda528563489, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,601 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9eb0364d53b384c3e59ecc15348e6c3c, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,601 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd97d8f813bdfcba3626cc4950c7c1e1, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,601 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5fb158c8201cfbc5babdbfc33ee45ab1, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,601 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbfd5210cf46b2a246edeeebd244b662, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,601 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94c604fd0d435e28ac643b49d6cbee14, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,601 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d72691fc99dffac743de3d6bcb0f84ed, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,601 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99e7b3565f08f10e8c368072e9c21c09, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,601 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=477e65b3290a1e6e03e5eb59f7aa956d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,601 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5017ae3acb74fe8c7d0b9c6d9a2cc402, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,601 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d5b1f002057972e76beae8f19094030, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,601 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f55c025b8995c15117005602a775de64, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,601 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed3111e35326fc2c5455ae2475ab1b5, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,601 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec3a5c93577675c61d946ed41b85bdb5, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,602 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ef5fca53c8ff0d12fc112ab68731575, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=5 2024-11-21T08:13:42,602 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dfe569750c256f741c6d3e5ed072503, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,602 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8202a2e0e04a2c5c7f5f0e491d40e, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,602 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=021eaa39c912c55ccf9b758a5c92d1f8, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,602 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af43494f9c57dd49b6d504756610f96c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,602 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3589ece79012bcc3080c57b249902675, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,602 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T08:13:42,602 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732176882602 2024-11-21T08:13:42,602 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732176942602 2024-11-21T08:13:42,602 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-21T08:13:42,602 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T08:13:42,602 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,602 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T08:13:42,602 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T08:13:42,603 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T08:13:42,603 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T08:13:42,603 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T08:13:42,603 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T08:13:42,604 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T08:13:42,605 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T08:13:42,605 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,605 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T08:13:42,605 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T08:13:42,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T08:13:42,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, REOPEN/MOVE 2024-11-21T08:13:42,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true 2024-11-21T08:13:42,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,606 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,607 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,607 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,607 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-21T08:13:42,607 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=65 2024-11-21T08:13:42,607 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=hbase:meta, region=1588230740 2024-11-21T08:13:42,608 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ed56fb6cbf7,35295,1732176812677, state=OPEN 2024-11-21T08:13:42,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,614 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,614 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,614 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,615 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732176852615 2024-11-21T08:13:42,615 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T08:13:42,615 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,616 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T08:13:42,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd0a67dc199543d2675c547a7cb41702, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,620 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T08:13:42,620 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3773623806176f3d7f8efb3efe1b7937, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f38fd8b8defc002b7cb3de558717f27c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f43187c46588bff73d01ba7eb26a6f8, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd29f056dad8c6c37d0979434891505d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=829529fdccf9853d96113fa67713ffbc, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee7ef14c3df8915e550cfda528563489, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9eb0364d53b384c3e59ecc15348e6c3c, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,621 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd97d8f813bdfcba3626cc4950c7c1e1, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5fb158c8201cfbc5babdbfc33ee45ab1, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbfd5210cf46b2a246edeeebd244b662, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94c604fd0d435e28ac643b49d6cbee14, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d72691fc99dffac743de3d6bcb0f84ed, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99e7b3565f08f10e8c368072e9c21c09, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=477e65b3290a1e6e03e5eb59f7aa956d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5017ae3acb74fe8c7d0b9c6d9a2cc402, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d5b1f002057972e76beae8f19094030, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f55c025b8995c15117005602a775de64, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed3111e35326fc2c5455ae2475ab1b5, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec3a5c93577675c61d946ed41b85bdb5, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ef5fca53c8ff0d12fc112ab68731575, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=5 2024-11-21T08:13:42,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dfe569750c256f741c6d3e5ed072503, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,623 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8202a2e0e04a2c5c7f5f0e491d40e, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,623 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=021eaa39c912c55ccf9b758a5c92d1f8, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,623 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af43494f9c57dd49b6d504756610f96c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,623 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3589ece79012bcc3080c57b249902675, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,623 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T08:13:42,623 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732176882623 2024-11-21T08:13:42,623 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732176942623 2024-11-21T08:13:42,623 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-21T08:13:42,623 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T08:13:42,623 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,623 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T08:13:42,623 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T08:13:42,624 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T08:13:42,624 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T08:13:42,624 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T08:13:42,624 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T08:13:42,625 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T08:13:42,627 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T08:13:42,627 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T08:13:42,628 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T08:13:42,629 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T08:13:42,630 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, REOPEN/MOVE 2024-11-21T08:13:42,630 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true 2024-11-21T08:13:42,630 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,630 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,631 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 6ed56fb6cbf7%2C33091%2C1732176812968.meta.1732176815418.meta, worker=6ed56fb6cbf7,38845,1732176812866 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T08:13:42,631 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-21T08:13:42,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741843_1019 (size=45) 2024-11-21T08:13:42,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741846_1022 (size=45) 2024-11-21T08:13:42,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741844_1020 (size=45) 2024-11-21T08:13:42,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=65 2024-11-21T08:13:42,632 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=hbase:meta, region=1588230740 2024-11-21T08:13:42,632 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ed56fb6cbf7,35295,1732176812677, state=OPEN 2024-11-21T08:13:42,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741847_1023 (size=45) 2024-11-21T08:13:42,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,646 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,646 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,646 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,646 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732176852646 2024-11-21T08:13:42,647 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T08:13:42,647 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,647 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T08:13:42,648 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T08:13:42,648 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true 2024-11-21T08:13:42,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,649 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd0a67dc199543d2675c547a7cb41702, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3773623806176f3d7f8efb3efe1b7937, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f38fd8b8defc002b7cb3de558717f27c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f43187c46588bff73d01ba7eb26a6f8, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd29f056dad8c6c37d0979434891505d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=829529fdccf9853d96113fa67713ffbc, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee7ef14c3df8915e550cfda528563489, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9eb0364d53b384c3e59ecc15348e6c3c, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd97d8f813bdfcba3626cc4950c7c1e1, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5fb158c8201cfbc5babdbfc33ee45ab1, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbfd5210cf46b2a246edeeebd244b662, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94c604fd0d435e28ac643b49d6cbee14, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d72691fc99dffac743de3d6bcb0f84ed, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99e7b3565f08f10e8c368072e9c21c09, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=477e65b3290a1e6e03e5eb59f7aa956d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5017ae3acb74fe8c7d0b9c6d9a2cc402, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d5b1f002057972e76beae8f19094030, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f55c025b8995c15117005602a775de64, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed3111e35326fc2c5455ae2475ab1b5, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec3a5c93577675c61d946ed41b85bdb5, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ef5fca53c8ff0d12fc112ab68731575, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=5 2024-11-21T08:13:42,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dfe569750c256f741c6d3e5ed072503, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8202a2e0e04a2c5c7f5f0e491d40e, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=021eaa39c912c55ccf9b758a5c92d1f8, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af43494f9c57dd49b6d504756610f96c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3589ece79012bcc3080c57b249902675, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,655 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T08:13:42,655 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732176882655 2024-11-21T08:13:42,655 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732176942655 2024-11-21T08:13:42,655 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-21T08:13:42,655 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T08:13:42,655 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,655 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T08:13:42,655 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T08:13:42,655 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T08:13:42,655 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T08:13:42,656 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T08:13:42,656 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T08:13:42,657 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T08:13:42,658 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,658 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,659 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T08:13:42,659 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T08:13:42,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T08:13:42,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7ef5fca53c8ff0d12fc112ab68731575, REOPEN/MOVE 2024-11-21T08:13:42,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true 2024-11-21T08:13:42,660 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T08:13:42,660 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-21T08:13:42,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=65 2024-11-21T08:13:42,661 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ed56fb6cbf7,35295,1732176812677, table=hbase:meta, region=1588230740 2024-11-21T08:13:42,661 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ed56fb6cbf7,35295,1732176812677, state=OPEN 2024-11-21T08:13:42,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T08:13:42,733 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,733 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,733 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T08:13:42,733 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,734 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732176852734 2024-11-21T08:13:42,734 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T08:13:42,734 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "6ed56fb6cbf7" port: 38845 start_code: 1732176812866 } transition { transition_code: OPENED region_info { region_id: 1732176816148 table_name { namespace: "default" qualifier: "test" } start_key: "kkk" end_key: "lll" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 79 initiating_master_active_time: 1732176813431 }; retry (#1) after 2013ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-21T08:13:42,735 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,737 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T08:13:42,739 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=60, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure cd29f056dad8c6c37d0979434891505d, server=6ed56fb6cbf7,38845,1732176812866:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 6ed56fb6cbf7,33091,1732176812968, splitWal=true, meta=true exec-time=2.7970 sec 2024-11-21T08:13:42,739 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1924) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1288) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1214) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1264) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T08:13:42,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd0a67dc199543d2675c547a7cb41702, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3773623806176f3d7f8efb3efe1b7937, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f38fd8b8defc002b7cb3de558717f27c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f43187c46588bff73d01ba7eb26a6f8, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd29f056dad8c6c37d0979434891505d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=829529fdccf9853d96113fa67713ffbc, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee7ef14c3df8915e550cfda528563489, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9eb0364d53b384c3e59ecc15348e6c3c, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd97d8f813bdfcba3626cc4950c7c1e1, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5fb158c8201cfbc5babdbfc33ee45ab1, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbfd5210cf46b2a246edeeebd244b662, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94c604fd0d435e28ac643b49d6cbee14, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d72691fc99dffac743de3d6bcb0f84ed, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99e7b3565f08f10e8c368072e9c21c09, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=477e65b3290a1e6e03e5eb59f7aa956d, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5017ae3acb74fe8c7d0b9c6d9a2cc402, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d5b1f002057972e76beae8f19094030, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f55c025b8995c15117005602a775de64, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed3111e35326fc2c5455ae2475ab1b5, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec3a5c93577675c61d946ed41b85bdb5, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ef5fca53c8ff0d12fc112ab68731575, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=5 2024-11-21T08:13:42,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dfe569750c256f741c6d3e5ed072503, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8202a2e0e04a2c5c7f5f0e491d40e, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=021eaa39c912c55ccf9b758a5c92d1f8, regionState=OPENING, lastHost=6ed56fb6cbf7,33091,1732176812968, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af43494f9c57dd49b6d504756610f96c, regionState=OPEN, lastHost=6ed56fb6cbf7,38845,1732176812866, regionLocation=6ed56fb6cbf7,38845,1732176812866, openSeqNum=2 2024-11-21T08:13:42,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3589ece79012bcc3080c57b249902675, regionState=OPEN, lastHost=6ed56fb6cbf7,35295,1732176812677, regionLocation=6ed56fb6cbf7,35295,1732176812677, openSeqNum=2 2024-11-21T08:13:42,742 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32901 {}] ipc.CallRunner(138): callId: 94 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 126 connection: 172.17.0.3:42659 deadline: 1732176832739, exception=org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false 2024-11-21T08:13:42,743 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T08:13:42,743 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732176882743 2024-11-21T08:13:42,743 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732176942743 2024-11-21T08:13:42,743 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-21T08:13:42,743 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "6ed56fb6cbf7" port: 35295 start_code: 1732176812677 } transition { transition_code: OPENED region_info { region_id: 1732176816148 table_name { namespace: "default" qualifier: "test" } start_key: "hhh" end_key: "iii" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 81 initiating_master_active_time: 1732176813431 }; retry (#1) after 2000ms delay (Master is coming online...). org.apache.hadoop.hbase.PleaseHoldException: org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1924) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1288) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1214) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1264) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1924) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1288) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1214) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1264) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-21T08:13:42,759 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=352 (was 383), OpenFileDescriptor=594 (was 611), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=671 (was 671), ProcessCount=11 (was 11), AvailableMemoryMB=6005 (was 5440) - AvailableMemoryMB LEAK? - 2024-11-21T08:13:42,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-21T08:13:42,759 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T08:13:42,759 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T08:13:42,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T08:13:42,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T08:13:42,760 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T08:13:42,760 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-21T08:13:42,760 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=189905736, stopped=false 2024-11-21T08:13:42,760 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=6ed56fb6cbf7,32901,1732176811582 2024-11-21T08:13:42,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T08:13:42,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T08:13:42,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T08:13:42,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T08:13:42,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T08:13:42,817 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:42,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T08:13:42,817 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T08:13:42,817 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T08:13:42,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T08:13:42,817 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6ed56fb6cbf7,35295,1732176812677' ***** 2024-11-21T08:13:42,817 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T08:13:42,817 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6ed56fb6cbf7,38845,1732176812866' ***** 2024-11-21T08:13:42,817 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T08:13:42,818 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T08:13:42,818 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T08:13:42,818 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T08:13:42,818 INFO [RS:0;6ed56fb6cbf7:35295 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T08:13:42,818 INFO [RS:0;6ed56fb6cbf7:35295 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T08:13:42,818 INFO [RS:0;6ed56fb6cbf7:35295 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T08:13:42,818 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T08:13:42,818 INFO [RS:0;6ed56fb6cbf7:35295 {}] regionserver.HRegionServer(3091): Received CLOSE for 2d5b1f002057972e76beae8f19094030 2024-11-21T08:13:42,819 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T08:13:42,819 INFO [RS:1;6ed56fb6cbf7:38845 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T08:13:42,819 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T08:13:42,819 INFO [RS:0;6ed56fb6cbf7:35295 {}] regionserver.HRegionServer(3091): Received CLOSE for ee7ef14c3df8915e550cfda528563489 2024-11-21T08:13:42,819 INFO [RS:1;6ed56fb6cbf7:38845 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T08:13:42,819 INFO [RS:0;6ed56fb6cbf7:35295 {}] regionserver.HRegionServer(3091): Received CLOSE for 7ef5fca53c8ff0d12fc112ab68731575 2024-11-21T08:13:42,819 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.HRegionServer(3091): Received CLOSE for af43494f9c57dd49b6d504756610f96c 2024-11-21T08:13:42,819 INFO [RS:0;6ed56fb6cbf7:35295 {}] regionserver.HRegionServer(3091): Received CLOSE for 3589ece79012bcc3080c57b249902675 2024-11-21T08:13:42,819 INFO [RS:0;6ed56fb6cbf7:35295 {}] regionserver.HRegionServer(3091): Received CLOSE for 4f43187c46588bff73d01ba7eb26a6f8 2024-11-21T08:13:42,819 INFO [RS:0;6ed56fb6cbf7:35295 {}] regionserver.HRegionServer(3091): Received CLOSE for ec3a5c93577675c61d946ed41b85bdb5 2024-11-21T08:13:42,819 INFO [RS:0;6ed56fb6cbf7:35295 {}] regionserver.HRegionServer(3091): Received CLOSE for 99e7b3565f08f10e8c368072e9c21c09 2024-11-21T08:13:42,819 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.HRegionServer(3091): Received CLOSE for f55c025b8995c15117005602a775de64 2024-11-21T08:13:42,819 INFO [RS:0;6ed56fb6cbf7:35295 {}] regionserver.HRegionServer(3091): Received CLOSE for 5fb158c8201cfbc5babdbfc33ee45ab1 2024-11-21T08:13:42,819 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.HRegionServer(3091): Received CLOSE for 8dfe569750c256f741c6d3e5ed072503 2024-11-21T08:13:42,819 INFO [RS:0;6ed56fb6cbf7:35295 {}] regionserver.HRegionServer(3091): Received CLOSE for ebf8202a2e0e04a2c5c7f5f0e491d40e 2024-11-21T08:13:42,819 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.HRegionServer(3091): Received CLOSE for cd29f056dad8c6c37d0979434891505d 2024-11-21T08:13:42,819 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.HRegionServer(3091): Received CLOSE for 4ed3111e35326fc2c5455ae2475ab1b5 2024-11-21T08:13:42,819 INFO [RS:0;6ed56fb6cbf7:35295 {}] regionserver.HRegionServer(959): stopping server 6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:42,819 INFO [RS:0;6ed56fb6cbf7:35295 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T08:13:42,819 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.HRegionServer(3091): Received CLOSE for f38fd8b8defc002b7cb3de558717f27c 2024-11-21T08:13:42,819 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.HRegionServer(3091): Received CLOSE for fd0a67dc199543d2675c547a7cb41702 2024-11-21T08:13:42,819 INFO [RS:0;6ed56fb6cbf7:35295 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;6ed56fb6cbf7:35295. 2024-11-21T08:13:42,820 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.HRegionServer(3091): Received CLOSE for 5017ae3acb74fe8c7d0b9c6d9a2cc402 2024-11-21T08:13:42,820 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.HRegionServer(3091): Received CLOSE for 829529fdccf9853d96113fa67713ffbc 2024-11-21T08:13:42,820 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T08:13:42,820 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T08:13:42,820 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.HRegionServer(3091): Received CLOSE for 94c604fd0d435e28ac643b49d6cbee14 2024-11-21T08:13:42,820 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing af43494f9c57dd49b6d504756610f96c, disabling compactions & flushes 2024-11-21T08:13:42,820 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.HRegionServer(3091): Received CLOSE for d72691fc99dffac743de3d6bcb0f84ed 2024-11-21T08:13:42,820 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1732176816148.af43494f9c57dd49b6d504756610f96c. 2024-11-21T08:13:42,820 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.HRegionServer(3091): Received CLOSE for fd97d8f813bdfcba3626cc4950c7c1e1 2024-11-21T08:13:42,820 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732176816148.af43494f9c57dd49b6d504756610f96c. 2024-11-21T08:13:42,820 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.HRegionServer(959): stopping server 6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:42,820 INFO [RS:1;6ed56fb6cbf7:38845 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T08:13:42,820 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732176816148.af43494f9c57dd49b6d504756610f96c. after waiting 0 ms 2024-11-21T08:13:42,820 INFO [RS:0;6ed56fb6cbf7:35295 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T08:13:42,820 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732176816148.af43494f9c57dd49b6d504756610f96c. 2024-11-21T08:13:42,820 INFO [RS:1;6ed56fb6cbf7:38845 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;6ed56fb6cbf7:38845. 2024-11-21T08:13:42,820 INFO [RS:0;6ed56fb6cbf7:35295 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T08:13:42,820 INFO [RS:0;6ed56fb6cbf7:35295 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T08:13:42,820 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T08:13:42,820 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T08:13:42,820 INFO [RS:0;6ed56fb6cbf7:35295 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-21T08:13:42,820 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.HRegionServer(1321): Waiting on 12 regions to close 2024-11-21T08:13:42,820 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2d5b1f002057972e76beae8f19094030, disabling compactions & flushes 2024-11-21T08:13:42,820 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1732176816148.2d5b1f002057972e76beae8f19094030. 2024-11-21T08:13:42,821 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732176816148.2d5b1f002057972e76beae8f19094030. 2024-11-21T08:13:42,821 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732176816148.2d5b1f002057972e76beae8f19094030. after waiting 0 ms 2024-11-21T08:13:42,821 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732176816148.2d5b1f002057972e76beae8f19094030. 2024-11-21T08:13:42,821 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] regionserver.HRegionServer(1351): Waiting on 4ed3111e35326fc2c5455ae2475ab1b5, 5017ae3acb74fe8c7d0b9c6d9a2cc402, 829529fdccf9853d96113fa67713ffbc, 8dfe569750c256f741c6d3e5ed072503, 94c604fd0d435e28ac643b49d6cbee14, af43494f9c57dd49b6d504756610f96c, cd29f056dad8c6c37d0979434891505d, d72691fc99dffac743de3d6bcb0f84ed, dbfd5210cf46b2a246edeeebd244b662, f38fd8b8defc002b7cb3de558717f27c, f55c025b8995c15117005602a775de64, fd0a67dc199543d2675c547a7cb41702, fd97d8f813bdfcba3626cc4950c7c1e1 2024-11-21T08:13:42,822 INFO [RS:0;6ed56fb6cbf7:35295 {}] regionserver.HRegionServer(1321): Waiting on 10 regions to close 2024-11-21T08:13:42,822 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2d5b1f002057972e76beae8f19094030, 3589ece79012bcc3080c57b249902675, 4f43187c46588bff73d01ba7eb26a6f8, 5fb158c8201cfbc5babdbfc33ee45ab1, 7ef5fca53c8ff0d12fc112ab68731575, 99e7b3565f08f10e8c368072e9c21c09, 9eb0364d53b384c3e59ecc15348e6c3c, ebf8202a2e0e04a2c5c7f5f0e491d40e, ec3a5c93577675c61d946ed41b85bdb5, ee7ef14c3df8915e550cfda528563489 2024-11-21T08:13:42,822 DEBUG [RS_CLOSE_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T08:13:42,822 INFO [RS_CLOSE_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T08:13:42,822 DEBUG [RS_CLOSE_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T08:13:42,822 DEBUG [RS_CLOSE_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T08:13:42,822 DEBUG [RS_CLOSE_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T08:13:42,822 INFO [RS_CLOSE_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.64 KB heapSize=5.42 KB 2024-11-21T08:13:42,825 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/af43494f9c57dd49b6d504756610f96c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T08:13:42,826 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1732176816148.af43494f9c57dd49b6d504756610f96c. 2024-11-21T08:13:42,826 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for af43494f9c57dd49b6d504756610f96c: Waiting for close lock at 1732176822819Running coprocessor pre-close hooks at 1732176822820 (+1 ms)Disabling compacts and flushes for region at 1732176822820Disabling writes for close at 1732176822820Writing region close event to WAL at 1732176822821 (+1 ms)Running coprocessor post-close hooks at 1732176822826 (+5 ms)Closed at 1732176822826 2024-11-21T08:13:42,826 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1732176816148.af43494f9c57dd49b6d504756610f96c. 2024-11-21T08:13:42,826 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f55c025b8995c15117005602a775de64, disabling compactions & flushes 2024-11-21T08:13:42,826 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1732176816148.f55c025b8995c15117005602a775de64. 2024-11-21T08:13:42,826 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732176816148.f55c025b8995c15117005602a775de64. 2024-11-21T08:13:42,826 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732176816148.f55c025b8995c15117005602a775de64. after waiting 0 ms 2024-11-21T08:13:42,827 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732176816148.f55c025b8995c15117005602a775de64. 2024-11-21T08:13:42,827 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/2d5b1f002057972e76beae8f19094030/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T08:13:42,832 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/f55c025b8995c15117005602a775de64/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T08:13:42,832 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1732176816148.f55c025b8995c15117005602a775de64. 2024-11-21T08:13:42,832 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1732176816148.2d5b1f002057972e76beae8f19094030. 2024-11-21T08:13:42,833 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f55c025b8995c15117005602a775de64: Waiting for close lock at 1732176822826Running coprocessor pre-close hooks at 1732176822826Disabling compacts and flushes for region at 1732176822826Disabling writes for close at 1732176822827 (+1 ms)Writing region close event to WAL at 1732176822827Running coprocessor post-close hooks at 1732176822832 (+5 ms)Closed at 1732176822832 2024-11-21T08:13:42,833 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2d5b1f002057972e76beae8f19094030: Waiting for close lock at 1732176822820Running coprocessor pre-close hooks at 1732176822820Disabling compacts and flushes for region at 1732176822820Disabling writes for close at 1732176822821 (+1 ms)Writing region close event to WAL at 1732176822823 (+2 ms)Running coprocessor post-close hooks at 1732176822832 (+9 ms)Closed at 1732176822832 2024-11-21T08:13:42,833 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1732176816148.f55c025b8995c15117005602a775de64. 2024-11-21T08:13:42,833 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1732176816148.2d5b1f002057972e76beae8f19094030. 2024-11-21T08:13:42,833 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8dfe569750c256f741c6d3e5ed072503, disabling compactions & flushes 2024-11-21T08:13:42,833 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ee7ef14c3df8915e550cfda528563489, disabling compactions & flushes 2024-11-21T08:13:42,833 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1732176816148.8dfe569750c256f741c6d3e5ed072503. 2024-11-21T08:13:42,833 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1732176816148.ee7ef14c3df8915e550cfda528563489. 2024-11-21T08:13:42,833 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732176816148.8dfe569750c256f741c6d3e5ed072503. 2024-11-21T08:13:42,833 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732176816148.ee7ef14c3df8915e550cfda528563489. 2024-11-21T08:13:42,833 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732176816148.8dfe569750c256f741c6d3e5ed072503. after waiting 0 ms 2024-11-21T08:13:42,833 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732176816148.8dfe569750c256f741c6d3e5ed072503. 2024-11-21T08:13:42,833 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732176816148.ee7ef14c3df8915e550cfda528563489. after waiting 0 ms 2024-11-21T08:13:42,833 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732176816148.ee7ef14c3df8915e550cfda528563489. 2024-11-21T08:13:42,842 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/8dfe569750c256f741c6d3e5ed072503/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T08:13:42,842 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/ee7ef14c3df8915e550cfda528563489/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T08:13:42,842 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1732176816148.8dfe569750c256f741c6d3e5ed072503. 2024-11-21T08:13:42,843 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8dfe569750c256f741c6d3e5ed072503: Waiting for close lock at 1732176822833Running coprocessor pre-close hooks at 1732176822833Disabling compacts and flushes for region at 1732176822833Disabling writes for close at 1732176822833Writing region close event to WAL at 1732176822833Running coprocessor post-close hooks at 1732176822842 (+9 ms)Closed at 1732176822842 2024-11-21T08:13:42,843 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1732176816148.ee7ef14c3df8915e550cfda528563489. 2024-11-21T08:13:42,843 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1732176816148.8dfe569750c256f741c6d3e5ed072503. 2024-11-21T08:13:42,843 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ee7ef14c3df8915e550cfda528563489: Waiting for close lock at 1732176822833Running coprocessor pre-close hooks at 1732176822833Disabling compacts and flushes for region at 1732176822833Disabling writes for close at 1732176822833Writing region close event to WAL at 1732176822834 (+1 ms)Running coprocessor post-close hooks at 1732176822843 (+9 ms)Closed at 1732176822843 2024-11-21T08:13:42,843 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cd29f056dad8c6c37d0979434891505d, disabling compactions & flushes 2024-11-21T08:13:42,843 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1732176816148.cd29f056dad8c6c37d0979434891505d. 2024-11-21T08:13:42,843 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1732176816148.ee7ef14c3df8915e550cfda528563489. 2024-11-21T08:13:42,843 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732176816148.cd29f056dad8c6c37d0979434891505d. 2024-11-21T08:13:42,843 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732176816148.cd29f056dad8c6c37d0979434891505d. after waiting 0 ms 2024-11-21T08:13:42,843 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732176816148.cd29f056dad8c6c37d0979434891505d. 2024-11-21T08:13:42,843 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7ef5fca53c8ff0d12fc112ab68731575, disabling compactions & flushes 2024-11-21T08:13:42,843 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575. 2024-11-21T08:13:42,843 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575. 2024-11-21T08:13:42,843 DEBUG [RS_CLOSE_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/.tmp/info/0127a15ac45947269bec89d227f5ae68 is 138, key is test,bbb,1732176816148.3773623806176f3d7f8efb3efe1b7937./info:regioninfo/1732176821431/Put/seqid=0 2024-11-21T08:13:42,843 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575. after waiting 0 ms 2024-11-21T08:13:42,843 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575. 2024-11-21T08:13:42,848 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/7ef5fca53c8ff0d12fc112ab68731575/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-21T08:13:42,849 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/cd29f056dad8c6c37d0979434891505d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T08:13:42,849 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575. 2024-11-21T08:13:42,849 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7ef5fca53c8ff0d12fc112ab68731575: Waiting for close lock at 1732176822843Running coprocessor pre-close hooks at 1732176822843Disabling compacts and flushes for region at 1732176822843Disabling writes for close at 1732176822843Writing region close event to WAL at 1732176822844 (+1 ms)Running coprocessor post-close hooks at 1732176822849 (+5 ms)Closed at 1732176822849 2024-11-21T08:13:42,849 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1732176816148.7ef5fca53c8ff0d12fc112ab68731575. 2024-11-21T08:13:42,849 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3589ece79012bcc3080c57b249902675, disabling compactions & flushes 2024-11-21T08:13:42,849 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1732176816148.3589ece79012bcc3080c57b249902675. 2024-11-21T08:13:42,849 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732176816148.3589ece79012bcc3080c57b249902675. 2024-11-21T08:13:42,849 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1732176816148.cd29f056dad8c6c37d0979434891505d. 2024-11-21T08:13:42,849 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732176816148.3589ece79012bcc3080c57b249902675. after waiting 0 ms 2024-11-21T08:13:42,849 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732176816148.3589ece79012bcc3080c57b249902675. 2024-11-21T08:13:42,849 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cd29f056dad8c6c37d0979434891505d: Waiting for close lock at 1732176822843Running coprocessor pre-close hooks at 1732176822843Disabling compacts and flushes for region at 1732176822843Disabling writes for close at 1732176822843Writing region close event to WAL at 1732176822844 (+1 ms)Running coprocessor post-close hooks at 1732176822849 (+5 ms)Closed at 1732176822849 2024-11-21T08:13:42,850 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1732176816148.cd29f056dad8c6c37d0979434891505d. 2024-11-21T08:13:42,850 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4ed3111e35326fc2c5455ae2475ab1b5, disabling compactions & flushes 2024-11-21T08:13:42,850 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1732176816148.4ed3111e35326fc2c5455ae2475ab1b5. 2024-11-21T08:13:42,850 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732176816148.4ed3111e35326fc2c5455ae2475ab1b5. 2024-11-21T08:13:42,850 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732176816148.4ed3111e35326fc2c5455ae2475ab1b5. after waiting 0 ms 2024-11-21T08:13:42,850 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732176816148.4ed3111e35326fc2c5455ae2475ab1b5. 2024-11-21T08:13:42,857 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/3589ece79012bcc3080c57b249902675/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T08:13:42,858 INFO [regionserver/6ed56fb6cbf7:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T08:13:42,858 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1732176816148.3589ece79012bcc3080c57b249902675. 2024-11-21T08:13:42,858 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3589ece79012bcc3080c57b249902675: Waiting for close lock at 1732176822849Running coprocessor pre-close hooks at 1732176822849Disabling compacts and flushes for region at 1732176822849Disabling writes for close at 1732176822849Writing region close event to WAL at 1732176822852 (+3 ms)Running coprocessor post-close hooks at 1732176822858 (+6 ms)Closed at 1732176822858 2024-11-21T08:13:42,858 INFO [regionserver/6ed56fb6cbf7:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T08:13:42,858 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1732176816148.3589ece79012bcc3080c57b249902675. 2024-11-21T08:13:42,858 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4f43187c46588bff73d01ba7eb26a6f8, disabling compactions & flushes 2024-11-21T08:13:42,858 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1732176816148.4f43187c46588bff73d01ba7eb26a6f8. 2024-11-21T08:13:42,858 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732176816148.4f43187c46588bff73d01ba7eb26a6f8. 2024-11-21T08:13:42,858 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732176816148.4f43187c46588bff73d01ba7eb26a6f8. after waiting 0 ms 2024-11-21T08:13:42,858 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732176816148.4f43187c46588bff73d01ba7eb26a6f8. 2024-11-21T08:13:42,860 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/4ed3111e35326fc2c5455ae2475ab1b5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T08:13:42,861 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1732176816148.4ed3111e35326fc2c5455ae2475ab1b5. 2024-11-21T08:13:42,861 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4ed3111e35326fc2c5455ae2475ab1b5: Waiting for close lock at 1732176822850Running coprocessor pre-close hooks at 1732176822850Disabling compacts and flushes for region at 1732176822850Disabling writes for close at 1732176822850Writing region close event to WAL at 1732176822856 (+6 ms)Running coprocessor post-close hooks at 1732176822861 (+5 ms)Closed at 1732176822861 2024-11-21T08:13:42,861 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1732176816148.4ed3111e35326fc2c5455ae2475ab1b5. 2024-11-21T08:13:42,861 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f38fd8b8defc002b7cb3de558717f27c, disabling compactions & flushes 2024-11-21T08:13:42,861 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1732176816148.f38fd8b8defc002b7cb3de558717f27c. 2024-11-21T08:13:42,862 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732176816148.f38fd8b8defc002b7cb3de558717f27c. 2024-11-21T08:13:42,862 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732176816148.f38fd8b8defc002b7cb3de558717f27c. after waiting 0 ms 2024-11-21T08:13:42,862 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732176816148.f38fd8b8defc002b7cb3de558717f27c. 2024-11-21T08:13:42,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741869_1045 (size=8177) 2024-11-21T08:13:42,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741869_1045 (size=8177) 2024-11-21T08:13:42,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741869_1045 (size=8177) 2024-11-21T08:13:42,865 INFO [RS_CLOSE_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.64 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/.tmp/info/0127a15ac45947269bec89d227f5ae68 2024-11-21T08:13:42,866 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/4f43187c46588bff73d01ba7eb26a6f8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T08:13:42,867 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1732176816148.4f43187c46588bff73d01ba7eb26a6f8. 2024-11-21T08:13:42,867 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4f43187c46588bff73d01ba7eb26a6f8: Waiting for close lock at 1732176822858Running coprocessor pre-close hooks at 1732176822858Disabling compacts and flushes for region at 1732176822858Disabling writes for close at 1732176822858Writing region close event to WAL at 1732176822859 (+1 ms)Running coprocessor post-close hooks at 1732176822867 (+8 ms)Closed at 1732176822867 2024-11-21T08:13:42,867 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1732176816148.4f43187c46588bff73d01ba7eb26a6f8. 2024-11-21T08:13:42,868 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ec3a5c93577675c61d946ed41b85bdb5, disabling compactions & flushes 2024-11-21T08:13:42,868 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1732176816148.ec3a5c93577675c61d946ed41b85bdb5. 2024-11-21T08:13:42,868 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732176816148.ec3a5c93577675c61d946ed41b85bdb5. 2024-11-21T08:13:42,868 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732176816148.ec3a5c93577675c61d946ed41b85bdb5. after waiting 0 ms 2024-11-21T08:13:42,868 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732176816148.ec3a5c93577675c61d946ed41b85bdb5. 2024-11-21T08:13:42,870 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/f38fd8b8defc002b7cb3de558717f27c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T08:13:42,871 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1732176816148.f38fd8b8defc002b7cb3de558717f27c. 2024-11-21T08:13:42,871 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f38fd8b8defc002b7cb3de558717f27c: Waiting for close lock at 1732176822861Running coprocessor pre-close hooks at 1732176822861Disabling compacts and flushes for region at 1732176822861Disabling writes for close at 1732176822862 (+1 ms)Writing region close event to WAL at 1732176822865 (+3 ms)Running coprocessor post-close hooks at 1732176822871 (+6 ms)Closed at 1732176822871 2024-11-21T08:13:42,872 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1732176816148.f38fd8b8defc002b7cb3de558717f27c. 2024-11-21T08:13:42,872 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fd0a67dc199543d2675c547a7cb41702, disabling compactions & flushes 2024-11-21T08:13:42,872 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1732176816148.fd0a67dc199543d2675c547a7cb41702. 2024-11-21T08:13:42,872 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732176816148.fd0a67dc199543d2675c547a7cb41702. 2024-11-21T08:13:42,872 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1732176816148.fd0a67dc199543d2675c547a7cb41702. after waiting 0 ms 2024-11-21T08:13:42,872 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1732176816148.fd0a67dc199543d2675c547a7cb41702. 2024-11-21T08:13:42,872 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/ec3a5c93577675c61d946ed41b85bdb5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T08:13:42,874 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1732176816148.ec3a5c93577675c61d946ed41b85bdb5. 2024-11-21T08:13:42,874 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ec3a5c93577675c61d946ed41b85bdb5: Waiting for close lock at 1732176822868Running coprocessor pre-close hooks at 1732176822868Disabling compacts and flushes for region at 1732176822868Disabling writes for close at 1732176822868Writing region close event to WAL at 1732176822868Running coprocessor post-close hooks at 1732176822874 (+6 ms)Closed at 1732176822874 2024-11-21T08:13:42,874 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1732176816148.ec3a5c93577675c61d946ed41b85bdb5. 2024-11-21T08:13:42,874 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 99e7b3565f08f10e8c368072e9c21c09, disabling compactions & flushes 2024-11-21T08:13:42,874 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1732176816148.99e7b3565f08f10e8c368072e9c21c09. 2024-11-21T08:13:42,874 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732176816148.99e7b3565f08f10e8c368072e9c21c09. 2024-11-21T08:13:42,875 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732176816148.99e7b3565f08f10e8c368072e9c21c09. after waiting 0 ms 2024-11-21T08:13:42,875 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732176816148.99e7b3565f08f10e8c368072e9c21c09. 2024-11-21T08:13:42,877 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/fd0a67dc199543d2675c547a7cb41702/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T08:13:42,878 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1732176816148.fd0a67dc199543d2675c547a7cb41702. 2024-11-21T08:13:42,878 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fd0a67dc199543d2675c547a7cb41702: Waiting for close lock at 1732176822872Running coprocessor pre-close hooks at 1732176822872Disabling compacts and flushes for region at 1732176822872Disabling writes for close at 1732176822872Writing region close event to WAL at 1732176822873 (+1 ms)Running coprocessor post-close hooks at 1732176822878 (+5 ms)Closed at 1732176822878 2024-11-21T08:13:42,878 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1732176816148.fd0a67dc199543d2675c547a7cb41702. 2024-11-21T08:13:42,878 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5017ae3acb74fe8c7d0b9c6d9a2cc402, disabling compactions & flushes 2024-11-21T08:13:42,878 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1732176816148.5017ae3acb74fe8c7d0b9c6d9a2cc402. 2024-11-21T08:13:42,878 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732176816148.5017ae3acb74fe8c7d0b9c6d9a2cc402. 2024-11-21T08:13:42,878 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732176816148.5017ae3acb74fe8c7d0b9c6d9a2cc402. after waiting 0 ms 2024-11-21T08:13:42,878 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732176816148.5017ae3acb74fe8c7d0b9c6d9a2cc402. 2024-11-21T08:13:42,878 DEBUG [RS_CLOSE_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/.tmp/info/0127a15ac45947269bec89d227f5ae68 as hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/info/0127a15ac45947269bec89d227f5ae68 2024-11-21T08:13:42,880 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/99e7b3565f08f10e8c368072e9c21c09/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T08:13:42,881 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1732176816148.99e7b3565f08f10e8c368072e9c21c09. 2024-11-21T08:13:42,881 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 99e7b3565f08f10e8c368072e9c21c09: Waiting for close lock at 1732176822874Running coprocessor pre-close hooks at 1732176822874Disabling compacts and flushes for region at 1732176822874Disabling writes for close at 1732176822875 (+1 ms)Writing region close event to WAL at 1732176822876 (+1 ms)Running coprocessor post-close hooks at 1732176822881 (+5 ms)Closed at 1732176822881 2024-11-21T08:13:42,881 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1732176816148.99e7b3565f08f10e8c368072e9c21c09. 2024-11-21T08:13:42,881 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5fb158c8201cfbc5babdbfc33ee45ab1, disabling compactions & flushes 2024-11-21T08:13:42,881 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1732176816148.5fb158c8201cfbc5babdbfc33ee45ab1. 2024-11-21T08:13:42,881 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732176816148.5fb158c8201cfbc5babdbfc33ee45ab1. 2024-11-21T08:13:42,882 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732176816148.5fb158c8201cfbc5babdbfc33ee45ab1. after waiting 0 ms 2024-11-21T08:13:42,882 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732176816148.5fb158c8201cfbc5babdbfc33ee45ab1. 2024-11-21T08:13:42,889 INFO [regionserver/6ed56fb6cbf7:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-21T08:13:42,889 INFO [regionserver/6ed56fb6cbf7:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-21T08:13:42,891 INFO [RS_CLOSE_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/info/0127a15ac45947269bec89d227f5ae68, entries=24, sequenceid=75, filesize=8.0 K 2024-11-21T08:13:42,891 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/5fb158c8201cfbc5babdbfc33ee45ab1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T08:13:42,892 INFO [regionserver/6ed56fb6cbf7:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-21T08:13:42,892 INFO [regionserver/6ed56fb6cbf7:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-21T08:13:42,892 INFO [RS_CLOSE_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 70ms, sequenceid=75, compaction requested=false 2024-11-21T08:13:42,892 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1732176816148.5fb158c8201cfbc5babdbfc33ee45ab1. 2024-11-21T08:13:42,892 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5fb158c8201cfbc5babdbfc33ee45ab1: Waiting for close lock at 1732176822881Running coprocessor pre-close hooks at 1732176822881Disabling compacts and flushes for region at 1732176822881Disabling writes for close at 1732176822882 (+1 ms)Writing region close event to WAL at 1732176822884 (+2 ms)Running coprocessor post-close hooks at 1732176822892 (+8 ms)Closed at 1732176822892 2024-11-21T08:13:42,892 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1732176816148.5fb158c8201cfbc5babdbfc33ee45ab1. 2024-11-21T08:13:42,892 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ebf8202a2e0e04a2c5c7f5f0e491d40e, disabling compactions & flushes 2024-11-21T08:13:42,893 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1732176816148.ebf8202a2e0e04a2c5c7f5f0e491d40e. 2024-11-21T08:13:42,893 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732176816148.ebf8202a2e0e04a2c5c7f5f0e491d40e. 2024-11-21T08:13:42,893 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1732176816148.ebf8202a2e0e04a2c5c7f5f0e491d40e. after waiting 0 ms 2024-11-21T08:13:42,893 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1732176816148.ebf8202a2e0e04a2c5c7f5f0e491d40e. 2024-11-21T08:13:42,906 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/5017ae3acb74fe8c7d0b9c6d9a2cc402/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T08:13:42,906 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1732176816148.5017ae3acb74fe8c7d0b9c6d9a2cc402. 2024-11-21T08:13:42,907 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5017ae3acb74fe8c7d0b9c6d9a2cc402: Waiting for close lock at 1732176822878Running coprocessor pre-close hooks at 1732176822878Disabling compacts and flushes for region at 1732176822878Disabling writes for close at 1732176822878Writing region close event to WAL at 1732176822879 (+1 ms)Running coprocessor post-close hooks at 1732176822906 (+27 ms)Closed at 1732176822906 2024-11-21T08:13:42,907 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1732176816148.5017ae3acb74fe8c7d0b9c6d9a2cc402. 2024-11-21T08:13:42,907 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 829529fdccf9853d96113fa67713ffbc, disabling compactions & flushes 2024-11-21T08:13:42,907 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1732176816148.829529fdccf9853d96113fa67713ffbc. 2024-11-21T08:13:42,907 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732176816148.829529fdccf9853d96113fa67713ffbc. 2024-11-21T08:13:42,907 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732176816148.829529fdccf9853d96113fa67713ffbc. after waiting 0 ms 2024-11-21T08:13:42,907 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732176816148.829529fdccf9853d96113fa67713ffbc. 2024-11-21T08:13:42,909 DEBUG [RS_CLOSE_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/hbase/meta/1588230740/recovered.edits/78.seqid, newMaxSeqId=78, maxSeqId=64 2024-11-21T08:13:42,909 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/ebf8202a2e0e04a2c5c7f5f0e491d40e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T08:13:42,910 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1732176816148.ebf8202a2e0e04a2c5c7f5f0e491d40e. 2024-11-21T08:13:42,910 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ebf8202a2e0e04a2c5c7f5f0e491d40e: Waiting for close lock at 1732176822892Running coprocessor pre-close hooks at 1732176822892Disabling compacts and flushes for region at 1732176822892Disabling writes for close at 1732176822893 (+1 ms)Writing region close event to WAL at 1732176822894 (+1 ms)Running coprocessor post-close hooks at 1732176822910 (+16 ms)Closed at 1732176822910 2024-11-21T08:13:42,910 DEBUG [RS_CLOSE_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T08:13:42,910 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1732176816148.ebf8202a2e0e04a2c5c7f5f0e491d40e. 2024-11-21T08:13:42,910 INFO [RS_CLOSE_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T08:13:42,910 DEBUG [RS_CLOSE_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732176822822Running coprocessor pre-close hooks at 1732176822822Disabling compacts and flushes for region at 1732176822822Disabling writes for close at 1732176822822Obtaining lock to block concurrent updates at 1732176822822Preparing flush snapshotting stores in 1588230740 at 1732176822822Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2708, getHeapSize=5488, getOffHeapSize=0, getCellsCount=24 at 1732176822829 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732176822832 (+3 ms)Flushing 1588230740/info: creating writer at 1732176822832Flushing 1588230740/info: appending metadata at 1732176822843 (+11 ms)Flushing 1588230740/info: closing flushed file at 1732176822843Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d57ec1d: reopening flushed file at 1732176822877 (+34 ms)Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 70ms, sequenceid=75, compaction requested=false at 1732176822892 (+15 ms)Writing region close event to WAL at 1732176822895 (+3 ms)Running coprocessor post-close hooks at 1732176822910 (+15 ms)Closed at 1732176822910 2024-11-21T08:13:42,910 DEBUG [RS_CLOSE_META-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-21T08:13:42,913 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/829529fdccf9853d96113fa67713ffbc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T08:13:42,913 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1732176816148.829529fdccf9853d96113fa67713ffbc. 2024-11-21T08:13:42,913 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 829529fdccf9853d96113fa67713ffbc: Waiting for close lock at 1732176822907Running coprocessor pre-close hooks at 1732176822907Disabling compacts and flushes for region at 1732176822907Disabling writes for close at 1732176822907Writing region close event to WAL at 1732176822908 (+1 ms)Running coprocessor post-close hooks at 1732176822913 (+5 ms)Closed at 1732176822913 2024-11-21T08:13:42,914 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1732176816148.829529fdccf9853d96113fa67713ffbc. 2024-11-21T08:13:42,914 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 94c604fd0d435e28ac643b49d6cbee14, disabling compactions & flushes 2024-11-21T08:13:42,914 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1732176816148.94c604fd0d435e28ac643b49d6cbee14. 2024-11-21T08:13:42,914 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732176816148.94c604fd0d435e28ac643b49d6cbee14. 2024-11-21T08:13:42,914 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732176816148.94c604fd0d435e28ac643b49d6cbee14. after waiting 0 ms 2024-11-21T08:13:42,914 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732176816148.94c604fd0d435e28ac643b49d6cbee14. 2024-11-21T08:13:42,921 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/94c604fd0d435e28ac643b49d6cbee14/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T08:13:42,922 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1732176816148.94c604fd0d435e28ac643b49d6cbee14. 2024-11-21T08:13:42,922 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 94c604fd0d435e28ac643b49d6cbee14: Waiting for close lock at 1732176822914Running coprocessor pre-close hooks at 1732176822914Disabling compacts and flushes for region at 1732176822914Disabling writes for close at 1732176822914Writing region close event to WAL at 1732176822914Running coprocessor post-close hooks at 1732176822922 (+8 ms)Closed at 1732176822922 2024-11-21T08:13:42,922 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1732176816148.94c604fd0d435e28ac643b49d6cbee14. 2024-11-21T08:13:42,922 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d72691fc99dffac743de3d6bcb0f84ed, disabling compactions & flushes 2024-11-21T08:13:42,922 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1732176816148.d72691fc99dffac743de3d6bcb0f84ed. 2024-11-21T08:13:42,922 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732176816148.d72691fc99dffac743de3d6bcb0f84ed. 2024-11-21T08:13:42,922 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732176816148.d72691fc99dffac743de3d6bcb0f84ed. after waiting 0 ms 2024-11-21T08:13:42,922 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732176816148.d72691fc99dffac743de3d6bcb0f84ed. 2024-11-21T08:13:42,930 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/d72691fc99dffac743de3d6bcb0f84ed/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T08:13:42,931 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1732176816148.d72691fc99dffac743de3d6bcb0f84ed. 2024-11-21T08:13:42,931 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d72691fc99dffac743de3d6bcb0f84ed: Waiting for close lock at 1732176822922Running coprocessor pre-close hooks at 1732176822922Disabling compacts and flushes for region at 1732176822922Disabling writes for close at 1732176822922Writing region close event to WAL at 1732176822925 (+3 ms)Running coprocessor post-close hooks at 1732176822931 (+6 ms)Closed at 1732176822931 2024-11-21T08:13:42,931 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1732176816148.d72691fc99dffac743de3d6bcb0f84ed. 2024-11-21T08:13:42,931 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fd97d8f813bdfcba3626cc4950c7c1e1, disabling compactions & flushes 2024-11-21T08:13:42,931 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1732176816148.fd97d8f813bdfcba3626cc4950c7c1e1. 2024-11-21T08:13:42,931 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732176816148.fd97d8f813bdfcba3626cc4950c7c1e1. 2024-11-21T08:13:42,931 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732176816148.fd97d8f813bdfcba3626cc4950c7c1e1. after waiting 0 ms 2024-11-21T08:13:42,931 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732176816148.fd97d8f813bdfcba3626cc4950c7c1e1. 2024-11-21T08:13:42,939 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/data/default/test/fd97d8f813bdfcba3626cc4950c7c1e1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T08:13:42,940 INFO [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1732176816148.fd97d8f813bdfcba3626cc4950c7c1e1. 2024-11-21T08:13:42,940 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fd97d8f813bdfcba3626cc4950c7c1e1: Waiting for close lock at 1732176822931Running coprocessor pre-close hooks at 1732176822931Disabling compacts and flushes for region at 1732176822931Disabling writes for close at 1732176822931Writing region close event to WAL at 1732176822932 (+1 ms)Running coprocessor post-close hooks at 1732176822940 (+8 ms)Closed at 1732176822940 2024-11-21T08:13:42,940 DEBUG [RS_CLOSE_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1732176816148.fd97d8f813bdfcba3626cc4950c7c1e1. 2024-11-21T08:13:43,021 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.HRegionServer(976): stopping server 6ed56fb6cbf7,38845,1732176812866; all regions closed. 2024-11-21T08:13:43,022 INFO [RS:0;6ed56fb6cbf7:35295 {}] regionserver.HRegionServer(976): stopping server 6ed56fb6cbf7,35295,1732176812677; all regions closed. 2024-11-21T08:13:43,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741835_1011 (size=6795) 2024-11-21T08:13:43,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741835_1011 (size=6795) 2024-11-21T08:13:43,028 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/oldWALs 2024-11-21T08:13:43,028 INFO [RS:1;6ed56fb6cbf7:38845 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6ed56fb6cbf7%2C38845%2C1732176812866:(num 1732176815010) 2024-11-21T08:13:43,028 DEBUG [RS:1;6ed56fb6cbf7:38845 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T08:13:43,028 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T08:13:43,029 INFO [RS:1;6ed56fb6cbf7:38845 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T08:13:43,029 INFO [RS:1;6ed56fb6cbf7:38845 {}] hbase.ChoreService(370): Chore service for: regionserver/6ed56fb6cbf7:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T08:13:43,029 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T08:13:43,029 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T08:13:43,029 INFO [regionserver/6ed56fb6cbf7:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T08:13:43,029 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T08:13:43,030 INFO [RS:1;6ed56fb6cbf7:38845 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T08:13:43,030 INFO [RS:1;6ed56fb6cbf7:38845 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38845 2024-11-21T08:13:43,030 WARN [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2374) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T08:13:43,030 WARN [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,kkk,1732176816148.dbfd5210cf46b2a246edeeebd244b662., aborting... java.io.IOException: Failed to report opened region to master: test,kkk,1732176816148.dbfd5210cf46b2a246edeeebd244b662. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T08:13:43,031 ERROR [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2440): ***** ABORTING region server 6ed56fb6cbf7,38845,1732176812866: Failed to open region test,kkk,1732176816148.dbfd5210cf46b2a246edeeebd244b662. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,kkk,1732176816148.dbfd5210cf46b2a246edeeebd244b662. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T08:13:43,031 ERROR [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-21T08:13:43,031 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-21T08:13:43,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741865_1041 (size=4192) 2024-11-21T08:13:43,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741865_1041 (size=4192) 2024-11-21T08:13:43,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741865_1041 (size=4192) 2024-11-21T08:13:43,037 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-21T08:13:43,039 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-21T08:13:43,040 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-21T08:13:43,042 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/oldWALs 2024-11-21T08:13:43,042 INFO [RS:0;6ed56fb6cbf7:35295 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6ed56fb6cbf7%2C35295%2C1732176812677.meta:.meta(num 1732176820592) 2024-11-21T08:13:43,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741834_1010 (size=5194) 2024-11-21T08:13:43,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741834_1010 (size=5194) 2024-11-21T08:13:43,050 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "Verbose": false, "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 508026408 }, "NonHeapMemoryUsage": { "committed": 164560896, "init": 7667712, "max": -1, "used": 161973872 }, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "6ed56fb6cbf7", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 2, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 35786, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 104, "ProcessCallTime_min": 0, "ProcessCallTime_max": 19, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 4, "ProcessCallTime_median": 9, "ProcessCallTime_75th_percentile": 14, "ProcessCallTime_90th_percentile": 17, "ProcessCallTime_95th_percentile": 18, "ProcessCallTime_98th_percentile": 18, "ProcessCallTime_99th_percentile": 18, "ProcessCallTime_99.9th_percentile": 18, "ProcessCallTime_TimeRangeCount_0-1": 104, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 104, "QueueCallTime_min": 0, "QueueCallTime_max": 10, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 2, "QueueCallTime_median": 5, "QueueCallTime_75th_percentile": 7, "QueueCallTime_90th_percentile": 9, "QueueCallTime_95th_percentile": 9, "QueueCallTime_98th_percentile": 9, "QueueCallTime_99th_percentile": 9, "QueueCallTime_99.9th_percentile": 9, "QueueCallTime_TimeRangeCount_0-1": 104, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 104, "TotalCallTime_min": 0, "TotalCallTime_max": 21, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 5, "TotalCallTime_median": 10, "TotalCallTime_75th_percentile": 15, "TotalCallTime_90th_percentile": 18, "TotalCallTime_95th_percentile": 19, "TotalCallTime_98th_percentile": 20, "TotalCallTime_99th_percentile": 20, "TotalCallTime_99.9th_percentile": 20, "TotalCallTime_TimeRangeCount_0-1": 104, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 104, "ResponseSize_min": 0, "ResponseSize_max": 22359, "ResponseSize_mean": 657, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16769, "ResponseSize_90th_percentile": 20123, "ResponseSize_95th_percentile": 21241, "ResponseSize_98th_percentile": 21911, "ResponseSize_99th_percentile": 22135, "ResponseSize_99.9th_percentile": 22336, "ResponseSize_SizeRangeCount_0-10": 104, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 104, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 313, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 104, "sentBytes": 54772 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "6ed56fb6cbf7", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:52614", "tag.serverName": "6ed56fb6cbf7,35295,1732176812677", "tag.clusterId": "5fd69682-7f93-4bcf-93cb-dc9c05c86979", "tag.Context": "regionserver", "tag.Hostname": "6ed56fb6cbf7", "regionCount": 10, "storeCount": 9, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2304, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732176812677, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3327, "localBytesRead": 3327, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.6, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 7446, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 8, "totalRowActionRequestCount": 8, "readRequestCount": 8, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 8, "rpcScanRequestCount": 8, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 8, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 8, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 46689, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 30111, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 46689, "FlushOutputSize_max": 46689, "FlushOutputSize_mean": 46689, "FlushOutputSize_25th_percentile": 46689, "FlushOutputSize_median": 46689, "FlushOutputSize_75th_percentile": 46689, "FlushOutputSize_90th_percentile": 46689, "FlushOutputSize_95th_percentile": 46689, "FlushOutputSize_98th_percentile": 46689, "FlushOutputSize_99th_percentile": 46689, "FlushOutputSize_99.9th_percentile": 46689, "FlushOutputSize_SizeRangeCount_0-10": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 30111, "FlushMemstoreSize_max": 30111, "FlushMemstoreSize_mean": 30111, "FlushMemstoreSize_25th_percentile": 30111, "FlushMemstoreSize_median": 30111, "FlushMemstoreSize_75th_percentile": 30111, "FlushMemstoreSize_90th_percentile": 30111, "FlushMemstoreSize_95th_percentile": 30111, "FlushMemstoreSize_98th_percentile": 30111, "FlushMemstoreSize_99th_percentile": 30111, "FlushMemstoreSize_99.9th_percentile": 30111, "FlushMemstoreSize_SizeRangeCount_0-10": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 236, "FlushTime_max": 236, "FlushTime_mean": 236, "FlushTime_25th_percentile": 236, "FlushTime_median": 236, "FlushTime_75th_percentile": 236, "FlushTime_90th_percentile": 236, "FlushTime_95th_percentile": 236, "FlushTime_98th_percentile": 236, "FlushTime_99th_percentile": 236, "FlushTime_99.9th_percentile": 236, "FlushTime_TimeRangeCount_0-1": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 8, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 8, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-21T08:13:43,057 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/oldWALs 2024-11-21T08:13:43,057 INFO [RS:0;6ed56fb6cbf7:35295 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6ed56fb6cbf7%2C35295%2C1732176812677:(num 1732176815010) 2024-11-21T08:13:43,057 DEBUG [RS:0;6ed56fb6cbf7:35295 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T08:13:43,057 INFO [RS:0;6ed56fb6cbf7:35295 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T08:13:43,057 INFO [RS:0;6ed56fb6cbf7:35295 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T08:13:43,057 INFO [RS:0;6ed56fb6cbf7:35295 {}] hbase.ChoreService(370): Chore service for: regionserver/6ed56fb6cbf7:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-21T08:13:43,057 INFO [RS:0;6ed56fb6cbf7:35295 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T08:13:43,058 WARN [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2374) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T08:13:43,058 WARN [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,hhh,1732176816148.9eb0364d53b384c3e59ecc15348e6c3c., aborting... java.io.IOException: Failed to report opened region to master: test,hhh,1732176816148.9eb0364d53b384c3e59ecc15348e6c3c. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T08:13:43,058 ERROR [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2440): ***** ABORTING region server 6ed56fb6cbf7,35295,1732176812677: Failed to open region test,hhh,1732176816148.9eb0364d53b384c3e59ecc15348e6c3c. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,hhh,1732176816148.9eb0364d53b384c3e59ecc15348e6c3c. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T08:13:43,058 ERROR [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-21T08:13:43,058 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-21T08:13:43,059 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-21T08:13:43,059 INFO [RS:0;6ed56fb6cbf7:35295 {}] executor.ExecutorService(108): Executor-11-RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0 had [Event #38 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler), Event #39 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler), Event #40 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler)] on shutdown 2024-11-21T08:13:43,060 INFO [RS:0;6ed56fb6cbf7:35295 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35295 2024-11-21T08:13:43,060 INFO [regionserver/6ed56fb6cbf7:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T08:13:43,060 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-21T08:13:43,061 DEBUG [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-21T08:13:43,063 INFO [RS_OPEN_REGION-regionserver/6ed56fb6cbf7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "Verbose": false, "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 510123560 }, "NonHeapMemoryUsage": { "committed": 164888576, "init": 7667712, "max": -1, "used": 162224704 }, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "6ed56fb6cbf7", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 2, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 35786, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 104, "ProcessCallTime_min": 0, "ProcessCallTime_max": 19, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 4, "ProcessCallTime_median": 9, "ProcessCallTime_75th_percentile": 14, "ProcessCallTime_90th_percentile": 17, "ProcessCallTime_95th_percentile": 18, "ProcessCallTime_98th_percentile": 18, "ProcessCallTime_99th_percentile": 18, "ProcessCallTime_99.9th_percentile": 18, "ProcessCallTime_TimeRangeCount_0-1": 104, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 104, "QueueCallTime_min": 0, "QueueCallTime_max": 10, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 2, "QueueCallTime_median": 5, "QueueCallTime_75th_percentile": 7, "QueueCallTime_90th_percentile": 9, "QueueCallTime_95th_percentile": 9, "QueueCallTime_98th_percentile": 9, "QueueCallTime_99th_percentile": 9, "QueueCallTime_99.9th_percentile": 9, "QueueCallTime_TimeRangeCount_0-1": 104, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 104, "TotalCallTime_min": 0, "TotalCallTime_max": 21, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 5, "TotalCallTime_median": 10, "TotalCallTime_75th_percentile": 15, "TotalCallTime_90th_percentile": 18, "TotalCallTime_95th_percentile": 19, "TotalCallTime_98th_percentile": 20, "TotalCallTime_99th_percentile": 20, "TotalCallTime_99.9th_percentile": 20, "TotalCallTime_TimeRangeCount_0-1": 104, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 104, "ResponseSize_min": 0, "ResponseSize_max": 22359, "ResponseSize_mean": 657, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16769, "ResponseSize_90th_percentile": 20123, "ResponseSize_95th_percentile": 21241, "ResponseSize_98th_percentile": 21911, "ResponseSize_99th_percentile": 22135, "ResponseSize_99.9th_percentile": 22336, "ResponseSize_SizeRangeCount_0-10": 104, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 104, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 313, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 104, "sentBytes": 54772 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "6ed56fb6cbf7", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:52614", "tag.serverName": "6ed56fb6cbf7,35295,1732176812677", "tag.clusterId": "5fd69682-7f93-4bcf-93cb-dc9c05c86979", "tag.Context": "regionserver", "tag.Hostname": "6ed56fb6cbf7", "regionCount": 10, "storeCount": 9, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2304, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732176812677, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3327, "localBytesRead": 3327, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.6, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 7446, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 8, "totalRowActionRequestCount": 8, "readRequestCount": 8, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 8, "rpcScanRequestCount": 8, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 8, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 8, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 46689, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 30111, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 46689, "FlushOutputSize_max": 46689, "FlushOutputSize_mean": 46689, "FlushOutputSize_25th_percentile": 46689, "FlushOutputSize_median": 46689, "FlushOutputSize_75th_percentile": 46689, "FlushOutputSize_90th_percentile": 46689, "FlushOutputSize_95th_percentile": 46689, "FlushOutputSize_98th_percentile": 46689, "FlushOutputSize_99th_percentile": 46689, "FlushOutputSize_99.9th_percentile": 46689, "FlushOutputSize_SizeRangeCount_0-10": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 30111, "FlushMemstoreSize_max": 30111, "FlushMemstoreSize_mean": 30111, "FlushMemstoreSize_25th_percentile": 30111, "FlushMemstoreSize_median": 30111, "FlushMemstoreSize_75th_percentile": 30111, "FlushMemstoreSize_90th_percentile": 30111, "FlushMemstoreSize_95th_percentile": 30111, "FlushMemstoreSize_98th_percentile": 30111, "FlushMemstoreSize_99th_percentile": 30111, "FlushMemstoreSize_99.9th_percentile": 30111, "FlushMemstoreSize_SizeRangeCount_0-10": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 236, "FlushTime_max": 236, "FlushTime_mean": 236, "FlushTime_25th_percentile": 236, "FlushTime_median": 236, "FlushTime_75th_percentile": 236, "FlushTime_90th_percentile": 236, "FlushTime_95th_percentile": 236, "FlushTime_98th_percentile": 236, "FlushTime_99th_percentile": 236, "FlushTime_99.9th_percentile": 236, "FlushTime_TimeRangeCount_0-1": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 8, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 8, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-21T08:13:43,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T08:13:43,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6ed56fb6cbf7,38845,1732176812866 2024-11-21T08:13:43,085 INFO [RS:1;6ed56fb6cbf7:38845 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T08:13:43,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6ed56fb6cbf7,35295,1732176812677 2024-11-21T08:13:43,129 INFO [RS:0;6ed56fb6cbf7:35295 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T08:13:43,129 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6ed56fb6cbf7,38845,1732176812866] 2024-11-21T08:13:43,161 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6ed56fb6cbf7,38845,1732176812866 already deleted, retry=false 2024-11-21T08:13:43,161 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 6ed56fb6cbf7,38845,1732176812866 expired; onlineServers=1 2024-11-21T08:13:43,161 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6ed56fb6cbf7,35295,1732176812677] 2024-11-21T08:13:43,171 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6ed56fb6cbf7,35295,1732176812677 already deleted, retry=false 2024-11-21T08:13:43,172 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 6ed56fb6cbf7,35295,1732176812677 expired; onlineServers=0 2024-11-21T08:13:43,172 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '6ed56fb6cbf7,32901,1732176811582' ***** 2024-11-21T08:13:43,172 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-21T08:13:43,172 INFO [M:0;6ed56fb6cbf7:32901 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T08:13:43,172 INFO [M:0;6ed56fb6cbf7:32901 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T08:13:43,172 DEBUG [M:0;6ed56fb6cbf7:32901 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-21T08:13:43,172 DEBUG [M:0;6ed56fb6cbf7:32901 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-21T08:13:43,172 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-21T08:13:43,172 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster-HFileCleaner.small.0-1732176814557 {}] cleaner.HFileCleaner(306): Exit Thread[master/6ed56fb6cbf7:0:becomeActiveMaster-HFileCleaner.small.0-1732176814557,5,FailOnTimeoutGroup] 2024-11-21T08:13:43,172 DEBUG [master/6ed56fb6cbf7:0:becomeActiveMaster-HFileCleaner.large.0-1732176814555 {}] cleaner.HFileCleaner(306): Exit Thread[master/6ed56fb6cbf7:0:becomeActiveMaster-HFileCleaner.large.0-1732176814555,5,FailOnTimeoutGroup] 2024-11-21T08:13:43,173 INFO [M:0;6ed56fb6cbf7:32901 {}] hbase.ChoreService(370): Chore service for: master/6ed56fb6cbf7:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-21T08:13:43,173 INFO [M:0;6ed56fb6cbf7:32901 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T08:13:43,173 DEBUG [M:0;6ed56fb6cbf7:32901 {}] master.HMaster(1795): Stopping service threads 2024-11-21T08:13:43,173 INFO [M:0;6ed56fb6cbf7:32901 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-21T08:13:43,173 INFO [M:0;6ed56fb6cbf7:32901 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T08:13:43,173 INFO [M:0;6ed56fb6cbf7:32901 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T08:13:43,173 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-21T08:13:43,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-21T08:13:43,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T08:13:43,183 DEBUG [M:0;6ed56fb6cbf7:32901 {}] zookeeper.ZKUtil(347): master:32901-0x1015c748eb50000, quorum=127.0.0.1:52614, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-21T08:13:43,183 WARN [M:0;6ed56fb6cbf7:32901 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-21T08:13:43,184 INFO [M:0;6ed56fb6cbf7:32901 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/.lastflushedseqids 2024-11-21T08:13:43,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741870_1046 (size=1492) 2024-11-21T08:13:43,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741870_1046 (size=1492) 2024-11-21T08:13:43,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741870_1046 (size=1492) 2024-11-21T08:13:43,251 INFO [RS:1;6ed56fb6cbf7:38845 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T08:13:43,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T08:13:43,251 INFO [RS:1;6ed56fb6cbf7:38845 {}] regionserver.HRegionServer(1031): Exiting; stopping=6ed56fb6cbf7,38845,1732176812866; zookeeper connection closed. 2024-11-21T08:13:43,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1015c748eb50002, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T08:13:43,251 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@8748b82 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@8748b82 2024-11-21T08:13:43,261 INFO [RS:0;6ed56fb6cbf7:35295 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T08:13:43,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T08:13:43,261 INFO [RS:0;6ed56fb6cbf7:35295 {}] regionserver.HRegionServer(1031): Exiting; stopping=6ed56fb6cbf7,35295,1732176812677; zookeeper connection closed. 2024-11-21T08:13:43,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35295-0x1015c748eb50001, quorum=127.0.0.1:52614, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T08:13:43,262 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@35c7f845 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@35c7f845 2024-11-21T08:13:43,262 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-21T08:13:43,603 INFO [M:0;6ed56fb6cbf7:32901 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T08:13:43,603 INFO [M:0;6ed56fb6cbf7:32901 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-21T08:13:43,604 DEBUG [M:0;6ed56fb6cbf7:32901 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T08:13:43,604 INFO [M:0;6ed56fb6cbf7:32901 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T08:13:43,604 DEBUG [M:0;6ed56fb6cbf7:32901 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T08:13:43,604 DEBUG [M:0;6ed56fb6cbf7:32901 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T08:13:43,604 DEBUG [M:0;6ed56fb6cbf7:32901 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T08:13:43,604 INFO [M:0;6ed56fb6cbf7:32901 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=259.14 KB heapSize=314.98 KB 2024-11-21T08:13:43,630 DEBUG [M:0;6ed56fb6cbf7:32901 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/079f8a53850b417cbfb1d3b1ad83fea9 is 82, key is hbase:meta,,1/info:regioninfo/1732176820916/Put/seqid=0 2024-11-21T08:13:43,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34217 is added to blk_1073741871_1047 (size=6071) 2024-11-21T08:13:43,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741871_1047 (size=6071) 2024-11-21T08:13:43,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33295 is added to blk_1073741871_1047 (size=6071) 2024-11-21T08:13:43,642 INFO [M:0;6ed56fb6cbf7:32901 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/079f8a53850b417cbfb1d3b1ad83fea9 2024-11-21T08:13:43,677 DEBUG [M:0;6ed56fb6cbf7:32901 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45661/user/jenkins/test-data/75d55549-7d93-5a29-5a7a-c92bffccb3db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b5aa1f3b279d42f0b4102e6be89adf82 is 2987, key is \x00\x00\x00\x00\x00\x00\x00