2024-11-19 07:13:17,808 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-19 07:13:17,823 main DEBUG Took 0.012999 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-19 07:13:17,824 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-19 07:13:17,824 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-19 07:13:17,825 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-19 07:13:17,827 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 07:13:17,834 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-19 07:13:17,847 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 07:13:17,849 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 07:13:17,850 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 07:13:17,850 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 07:13:17,851 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 07:13:17,852 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 07:13:17,853 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 07:13:17,853 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 07:13:17,854 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 07:13:17,855 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 07:13:17,856 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 07:13:17,857 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 07:13:17,857 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 07:13:17,858 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 07:13:17,858 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 07:13:17,859 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 07:13:17,860 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 07:13:17,860 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 07:13:17,861 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 07:13:17,861 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 07:13:17,862 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 07:13:17,862 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 07:13:17,863 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 07:13:17,863 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 07:13:17,864 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 07:13:17,864 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-19 07:13:17,866 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 07:13:17,868 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-19 07:13:17,870 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-19 07:13:17,871 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-19 07:13:17,873 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-19 07:13:17,873 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-19 07:13:17,884 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-19 07:13:17,887 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-19 07:13:17,890 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-19 07:13:17,890 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-19 07:13:17,891 main DEBUG createAppenders(={Console}) 2024-11-19 07:13:17,892 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-19 07:13:17,892 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-19 07:13:17,893 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-19 07:13:17,893 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-19 07:13:17,894 main DEBUG OutputStream closed 2024-11-19 07:13:17,894 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-19 07:13:17,894 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-19 07:13:17,894 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-19 07:13:17,966 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-19 07:13:17,968 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-19 07:13:17,969 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-19 07:13:17,970 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-19 07:13:17,971 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-19 07:13:17,971 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-19 07:13:17,971 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-19 07:13:17,972 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-19 07:13:17,972 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-19 07:13:17,972 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-19 07:13:17,973 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-19 07:13:17,973 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-19 07:13:17,973 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-19 07:13:17,973 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-19 07:13:17,974 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-19 07:13:17,974 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-19 07:13:17,974 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-19 07:13:17,975 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-19 07:13:17,977 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19 07:13:17,978 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-19 07:13:17,978 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-19 07:13:17,979 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-19T07:13:17,991 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-19 07:13:17,993 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-19 07:13:17,994 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19T07:13:18,205 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e 2024-11-19T07:13:18,224 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-19T07:13:18,246 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/cluster_d6232770-b682-acd5-7b4b-de334cae59f7, deleteOnExit=true 2024-11-19T07:13:18,246 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-19T07:13:18,248 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/test.cache.data in system properties and HBase conf 2024-11-19T07:13:18,249 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/hadoop.tmp.dir in system properties and HBase conf 2024-11-19T07:13:18,250 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/hadoop.log.dir in system properties and HBase conf 2024-11-19T07:13:18,250 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-19T07:13:18,251 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-19T07:13:18,252 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-19T07:13:18,338 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-19T07:13:18,432 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T07:13:18,437 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-19T07:13:18,438 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-19T07:13:18,438 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-19T07:13:18,439 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T07:13:18,440 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-19T07:13:18,440 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-19T07:13:18,441 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T07:13:18,442 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T07:13:18,442 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-19T07:13:18,443 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/nfs.dump.dir in system properties and HBase conf 2024-11-19T07:13:18,444 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/java.io.tmpdir in system properties and HBase conf 2024-11-19T07:13:18,444 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T07:13:18,445 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-19T07:13:18,445 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-19T07:13:19,510 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-19T07:13:19,580 INFO [Time-limited test {}] log.Log(170): Logging initialized @2527ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-19T07:13:19,644 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T07:13:19,701 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T07:13:19,720 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T07:13:19,720 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T07:13:19,721 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-19T07:13:19,732 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T07:13:19,735 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/hadoop.log.dir/,AVAILABLE} 2024-11-19T07:13:19,736 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T07:13:19,949 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64418798{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/java.io.tmpdir/jetty-localhost-38045-hadoop-hdfs-3_4_1-tests_jar-_-any-16984885226325353700/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T07:13:19,964 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:38045} 2024-11-19T07:13:19,964 INFO [Time-limited test {}] server.Server(415): Started @2912ms 2024-11-19T07:13:20,667 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T07:13:20,675 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T07:13:20,681 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T07:13:20,681 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T07:13:20,681 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-19T07:13:20,688 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@51c39823{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/hadoop.log.dir/,AVAILABLE} 2024-11-19T07:13:20,689 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@c7135d9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T07:13:20,804 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@65e17246{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/java.io.tmpdir/jetty-localhost-42505-hadoop-hdfs-3_4_1-tests_jar-_-any-7949336086859866873/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T07:13:20,805 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@220c8d80{HTTP/1.1, (http/1.1)}{localhost:42505} 2024-11-19T07:13:20,805 INFO [Time-limited test {}] server.Server(415): Started @3753ms 2024-11-19T07:13:20,854 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-19T07:13:20,977 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T07:13:20,984 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T07:13:20,996 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T07:13:20,997 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T07:13:20,997 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-19T07:13:21,001 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2ff36c4c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/hadoop.log.dir/,AVAILABLE} 2024-11-19T07:13:21,002 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6331ed3f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T07:13:21,155 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@395ff33a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/java.io.tmpdir/jetty-localhost-44539-hadoop-hdfs-3_4_1-tests_jar-_-any-15270286431404851573/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T07:13:21,157 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4d24ac65{HTTP/1.1, (http/1.1)}{localhost:44539} 2024-11-19T07:13:21,157 INFO [Time-limited test {}] server.Server(415): Started @4105ms 2024-11-19T07:13:21,160 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-19T07:13:21,302 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T07:13:21,310 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T07:13:21,319 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T07:13:21,319 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T07:13:21,319 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-19T07:13:21,322 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6ed43cd1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/hadoop.log.dir/,AVAILABLE} 2024-11-19T07:13:21,323 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@48a2d1c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T07:13:21,466 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@46041a9d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/java.io.tmpdir/jetty-localhost-43027-hadoop-hdfs-3_4_1-tests_jar-_-any-4245225385947771255/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T07:13:21,470 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2a39ea25{HTTP/1.1, (http/1.1)}{localhost:43027} 2024-11-19T07:13:21,472 INFO [Time-limited test {}] server.Server(415): Started @4420ms 2024-11-19T07:13:21,479 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-19T07:13:22,856 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/cluster_d6232770-b682-acd5-7b4b-de334cae59f7/data/data4/current/BP-1298718631-172.17.0.2-1732000398981/current, will proceed with Du for space computation calculation, 2024-11-19T07:13:22,856 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/cluster_d6232770-b682-acd5-7b4b-de334cae59f7/data/data3/current/BP-1298718631-172.17.0.2-1732000398981/current, will proceed with Du for space computation calculation, 2024-11-19T07:13:22,856 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/cluster_d6232770-b682-acd5-7b4b-de334cae59f7/data/data1/current/BP-1298718631-172.17.0.2-1732000398981/current, will proceed with Du for space computation calculation, 2024-11-19T07:13:22,856 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/cluster_d6232770-b682-acd5-7b4b-de334cae59f7/data/data2/current/BP-1298718631-172.17.0.2-1732000398981/current, will proceed with Du for space computation calculation, 2024-11-19T07:13:22,893 WARN [Thread-136 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/cluster_d6232770-b682-acd5-7b4b-de334cae59f7/data/data5/current/BP-1298718631-172.17.0.2-1732000398981/current, will proceed with Du for space computation calculation, 2024-11-19T07:13:22,893 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-19T07:13:22,893 WARN [Thread-137 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/cluster_d6232770-b682-acd5-7b4b-de334cae59f7/data/data6/current/BP-1298718631-172.17.0.2-1732000398981/current, will proceed with Du for space computation calculation, 2024-11-19T07:13:22,893 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-19T07:13:22,912 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-19T07:13:22,940 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e 2024-11-19T07:13:22,941 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9d78e6470c8ff505 with lease ID 0x71e306d3ac7309fb: Processing first storage report for DS-78acb0be-8b81-4317-958b-6530bbb93025 from datanode DatanodeRegistration(127.0.0.1:39225, datanodeUuid=0a28c780-1e17-47aa-b808-58de0c15a61f, infoPort=32827, infoSecurePort=0, ipcPort=46403, storageInfo=lv=-57;cid=testClusterID;nsid=1291538349;c=1732000398981) 2024-11-19T07:13:22,942 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9d78e6470c8ff505 with lease ID 0x71e306d3ac7309fb: from storage DS-78acb0be-8b81-4317-958b-6530bbb93025 node DatanodeRegistration(127.0.0.1:39225, datanodeUuid=0a28c780-1e17-47aa-b808-58de0c15a61f, infoPort=32827, infoSecurePort=0, ipcPort=46403, storageInfo=lv=-57;cid=testClusterID;nsid=1291538349;c=1732000398981), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-19T07:13:22,942 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x71eec94212402d1e with lease ID 0x71e306d3ac7309fc: Processing first storage report for DS-e0056ec2-0746-4cc1-bce1-20ef1e1aeac2 from datanode DatanodeRegistration(127.0.0.1:41661, datanodeUuid=6ca70730-775e-4704-927c-6c349c717af9, infoPort=39091, infoSecurePort=0, ipcPort=46081, storageInfo=lv=-57;cid=testClusterID;nsid=1291538349;c=1732000398981) 2024-11-19T07:13:22,943 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x71eec94212402d1e with lease ID 0x71e306d3ac7309fc: from storage DS-e0056ec2-0746-4cc1-bce1-20ef1e1aeac2 node DatanodeRegistration(127.0.0.1:41661, datanodeUuid=6ca70730-775e-4704-927c-6c349c717af9, infoPort=39091, infoSecurePort=0, ipcPort=46081, storageInfo=lv=-57;cid=testClusterID;nsid=1291538349;c=1732000398981), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-19T07:13:22,943 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc92f34f7386db535 with lease ID 0x71e306d3ac7309fa: Processing first storage report for DS-671bf5ab-8128-4e2b-addb-a3ea7fdb8afc from datanode DatanodeRegistration(127.0.0.1:44773, datanodeUuid=be091ac1-d2a2-48dc-8322-4747d2c1b589, infoPort=33965, infoSecurePort=0, ipcPort=36885, storageInfo=lv=-57;cid=testClusterID;nsid=1291538349;c=1732000398981) 2024-11-19T07:13:22,943 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc92f34f7386db535 with lease ID 0x71e306d3ac7309fa: from storage DS-671bf5ab-8128-4e2b-addb-a3ea7fdb8afc node DatanodeRegistration(127.0.0.1:44773, datanodeUuid=be091ac1-d2a2-48dc-8322-4747d2c1b589, infoPort=33965, infoSecurePort=0, ipcPort=36885, storageInfo=lv=-57;cid=testClusterID;nsid=1291538349;c=1732000398981), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-19T07:13:22,943 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9d78e6470c8ff505 with lease ID 0x71e306d3ac7309fb: Processing first storage report for DS-4d2b0578-0680-446d-af91-e424da20c603 from datanode DatanodeRegistration(127.0.0.1:39225, datanodeUuid=0a28c780-1e17-47aa-b808-58de0c15a61f, infoPort=32827, infoSecurePort=0, ipcPort=46403, storageInfo=lv=-57;cid=testClusterID;nsid=1291538349;c=1732000398981) 2024-11-19T07:13:22,943 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9d78e6470c8ff505 with lease ID 0x71e306d3ac7309fb: from storage DS-4d2b0578-0680-446d-af91-e424da20c603 node DatanodeRegistration(127.0.0.1:39225, datanodeUuid=0a28c780-1e17-47aa-b808-58de0c15a61f, infoPort=32827, infoSecurePort=0, ipcPort=46403, storageInfo=lv=-57;cid=testClusterID;nsid=1291538349;c=1732000398981), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-19T07:13:22,944 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x71eec94212402d1e with lease ID 0x71e306d3ac7309fc: Processing first storage report for DS-47ee9373-0c5e-4d74-997f-a6fbc2f473a1 from datanode DatanodeRegistration(127.0.0.1:41661, datanodeUuid=6ca70730-775e-4704-927c-6c349c717af9, infoPort=39091, infoSecurePort=0, ipcPort=46081, storageInfo=lv=-57;cid=testClusterID;nsid=1291538349;c=1732000398981) 2024-11-19T07:13:22,944 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x71eec94212402d1e with lease ID 0x71e306d3ac7309fc: from storage DS-47ee9373-0c5e-4d74-997f-a6fbc2f473a1 node DatanodeRegistration(127.0.0.1:41661, datanodeUuid=6ca70730-775e-4704-927c-6c349c717af9, infoPort=39091, infoSecurePort=0, ipcPort=46081, storageInfo=lv=-57;cid=testClusterID;nsid=1291538349;c=1732000398981), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-19T07:13:22,944 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc92f34f7386db535 with lease ID 0x71e306d3ac7309fa: Processing first storage report for DS-ed33d7d9-4552-4494-b487-af6cfa740558 from datanode DatanodeRegistration(127.0.0.1:44773, datanodeUuid=be091ac1-d2a2-48dc-8322-4747d2c1b589, infoPort=33965, infoSecurePort=0, ipcPort=36885, storageInfo=lv=-57;cid=testClusterID;nsid=1291538349;c=1732000398981) 2024-11-19T07:13:22,945 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc92f34f7386db535 with lease ID 0x71e306d3ac7309fa: from storage DS-ed33d7d9-4552-4494-b487-af6cfa740558 node DatanodeRegistration(127.0.0.1:44773, datanodeUuid=be091ac1-d2a2-48dc-8322-4747d2c1b589, infoPort=33965, infoSecurePort=0, ipcPort=36885, storageInfo=lv=-57;cid=testClusterID;nsid=1291538349;c=1732000398981), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-19T07:13:23,010 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/cluster_d6232770-b682-acd5-7b4b-de334cae59f7/zookeeper_0, clientPort=53655, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/cluster_d6232770-b682-acd5-7b4b-de334cae59f7/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbf53f79-e81d-9cf7-b024-17d4d3fc928e/cluster_d6232770-b682-acd5-7b4b-de334cae59f7/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-19T07:13:23,020 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53655 2024-11-19T07:13:23,033 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T07:13:23,036 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T07:13:23,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741825_1001 (size=7) 2024-11-19T07:13:23,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741825_1001 (size=7) 2024-11-19T07:13:23,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741825_1001 (size=7) 2024-11-19T07:13:23,684 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d with version=8 2024-11-19T07:13:23,684 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/hbase-staging 2024-11-19T07:13:23,759 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-19T07:13:23,984 INFO [Time-limited test {}] client.ConnectionUtils(128): master/db0afee3eab9:0 server-side Connection retries=45 2024-11-19T07:13:23,993 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T07:13:23,993 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T07:13:23,997 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T07:13:23,998 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T07:13:23,998 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T07:13:24,127 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-19T07:13:24,180 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-19T07:13:24,189 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-19T07:13:24,193 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T07:13:24,217 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 2332 (auto-detected) 2024-11-19T07:13:24,218 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-19T07:13:24,236 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43779 2024-11-19T07:13:24,256 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43779 connecting to ZooKeeper ensemble=127.0.0.1:53655 2024-11-19T07:13:24,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:437790x0, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T07:13:24,401 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43779-0x10151f0c9da0000 connected 2024-11-19T07:13:24,495 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T07:13:24,501 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T07:13:24,509 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T07:13:24,513 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d, hbase.cluster.distributed=false 2024-11-19T07:13:24,535 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T07:13:24,539 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43779 2024-11-19T07:13:24,539 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43779 2024-11-19T07:13:24,540 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43779 2024-11-19T07:13:24,540 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43779 2024-11-19T07:13:24,541 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43779 2024-11-19T07:13:24,636 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/db0afee3eab9:0 server-side Connection retries=45 2024-11-19T07:13:24,637 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T07:13:24,637 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T07:13:24,638 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T07:13:24,638 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T07:13:24,638 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T07:13:24,640 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T07:13:24,643 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T07:13:24,644 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34425 2024-11-19T07:13:24,646 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34425 connecting to ZooKeeper ensemble=127.0.0.1:53655 2024-11-19T07:13:24,647 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T07:13:24,651 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T07:13:24,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:344250x0, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T07:13:24,670 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:344250x0, quorum=127.0.0.1:53655, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T07:13:24,670 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34425-0x10151f0c9da0001 connected 2024-11-19T07:13:24,673 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T07:13:24,681 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T07:13:24,683 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T07:13:24,689 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T07:13:24,690 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34425 2024-11-19T07:13:24,690 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34425 2024-11-19T07:13:24,691 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34425 2024-11-19T07:13:24,691 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34425 2024-11-19T07:13:24,692 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34425 2024-11-19T07:13:24,708 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/db0afee3eab9:0 server-side Connection retries=45 2024-11-19T07:13:24,708 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T07:13:24,709 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T07:13:24,709 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T07:13:24,709 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T07:13:24,710 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T07:13:24,710 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T07:13:24,710 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T07:13:24,711 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42435 2024-11-19T07:13:24,713 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42435 connecting to ZooKeeper ensemble=127.0.0.1:53655 2024-11-19T07:13:24,714 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T07:13:24,717 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T07:13:24,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:424350x0, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T07:13:24,734 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:424350x0, quorum=127.0.0.1:53655, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T07:13:24,734 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42435-0x10151f0c9da0002 connected 2024-11-19T07:13:24,735 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T07:13:24,736 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T07:13:24,737 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T07:13:24,739 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T07:13:24,739 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42435 2024-11-19T07:13:24,740 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42435 2024-11-19T07:13:24,740 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42435 2024-11-19T07:13:24,741 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42435 2024-11-19T07:13:24,741 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42435 2024-11-19T07:13:24,756 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/db0afee3eab9:0 server-side Connection retries=45 2024-11-19T07:13:24,757 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T07:13:24,757 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T07:13:24,757 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T07:13:24,757 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T07:13:24,757 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T07:13:24,758 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T07:13:24,758 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T07:13:24,759 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37853 2024-11-19T07:13:24,760 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37853 connecting to ZooKeeper ensemble=127.0.0.1:53655 2024-11-19T07:13:24,761 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T07:13:24,764 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T07:13:24,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:378530x0, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T07:13:24,776 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:378530x0, quorum=127.0.0.1:53655, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T07:13:24,776 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37853-0x10151f0c9da0003 connected 2024-11-19T07:13:24,776 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T07:13:24,777 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T07:13:24,778 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37853-0x10151f0c9da0003, quorum=127.0.0.1:53655, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T07:13:24,780 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37853-0x10151f0c9da0003, quorum=127.0.0.1:53655, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T07:13:24,781 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37853 2024-11-19T07:13:24,781 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37853 2024-11-19T07:13:24,781 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37853 2024-11-19T07:13:24,782 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37853 2024-11-19T07:13:24,782 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37853 2024-11-19T07:13:24,795 DEBUG [M:0;db0afee3eab9:43779 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;db0afee3eab9:43779 2024-11-19T07:13:24,796 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/db0afee3eab9,43779,1732000403834 2024-11-19T07:13:24,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T07:13:24,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x10151f0c9da0003, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T07:13:24,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T07:13:24,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T07:13:24,809 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/db0afee3eab9,43779,1732000403834 2024-11-19T07:13:24,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x10151f0c9da0003, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T07:13:24,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T07:13:24,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T07:13:24,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T07:13:24,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x10151f0c9da0003, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T07:13:24,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T07:13:24,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T07:13:24,840 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-19T07:13:24,841 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/db0afee3eab9,43779,1732000403834 from backup master directory 2024-11-19T07:13:24,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T07:13:24,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/db0afee3eab9,43779,1732000403834 2024-11-19T07:13:24,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T07:13:24,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x10151f0c9da0003, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T07:13:24,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T07:13:24,850 WARN [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T07:13:24,850 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=db0afee3eab9,43779,1732000403834 2024-11-19T07:13:24,852 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-19T07:13:24,853 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-19T07:13:24,905 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/hbase.id] with ID: 1a1c4a15-1b2b-4e90-9845-b6cc07547cfd 2024-11-19T07:13:24,905 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/.tmp/hbase.id 2024-11-19T07:13:24,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741826_1002 (size=42) 2024-11-19T07:13:24,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741826_1002 (size=42) 2024-11-19T07:13:24,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741826_1002 (size=42) 2024-11-19T07:13:24,920 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/.tmp/hbase.id]:[hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/hbase.id] 2024-11-19T07:13:24,966 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T07:13:24,973 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-19T07:13:24,991 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-19T07:13:25,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T07:13:25,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T07:13:25,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T07:13:25,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x10151f0c9da0003, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T07:13:25,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741827_1003 (size=196) 2024-11-19T07:13:25,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741827_1003 (size=196) 2024-11-19T07:13:25,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741827_1003 (size=196) 2024-11-19T07:13:25,044 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T07:13:25,046 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-19T07:13:25,058 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T07:13:25,061 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T07:13:25,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741828_1004 (size=1189) 2024-11-19T07:13:25,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741828_1004 (size=1189) 2024-11-19T07:13:25,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741828_1004 (size=1189) 2024-11-19T07:13:25,116 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/MasterData/data/master/store 2024-11-19T07:13:25,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741829_1005 (size=34) 2024-11-19T07:13:25,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741829_1005 (size=34) 2024-11-19T07:13:25,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741829_1005 (size=34) 2024-11-19T07:13:25,140 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-19T07:13:25,143 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:25,144 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T07:13:25,145 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T07:13:25,145 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T07:13:25,146 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T07:13:25,147 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T07:13:25,147 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T07:13:25,148 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732000405144Disabling compacts and flushes for region at 1732000405144Disabling writes for close at 1732000405146 (+2 ms)Writing region close event to WAL at 1732000405147 (+1 ms)Closed at 1732000405147 2024-11-19T07:13:25,150 WARN [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/MasterData/data/master/store/.initializing 2024-11-19T07:13:25,150 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/MasterData/WALs/db0afee3eab9,43779,1732000403834 2024-11-19T07:13:25,157 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T07:13:25,170 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db0afee3eab9%2C43779%2C1732000403834, suffix=, logDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/MasterData/WALs/db0afee3eab9,43779,1732000403834, archiveDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/MasterData/oldWALs, maxLogs=10 2024-11-19T07:13:25,190 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/MasterData/WALs/db0afee3eab9,43779,1732000403834/db0afee3eab9%2C43779%2C1732000403834.1732000405174, exclude list is [], retry=0 2024-11-19T07:13:25,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41661,DS-e0056ec2-0746-4cc1-bce1-20ef1e1aeac2,DISK] 2024-11-19T07:13:25,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39225,DS-78acb0be-8b81-4317-958b-6530bbb93025,DISK] 2024-11-19T07:13:25,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44773,DS-671bf5ab-8128-4e2b-addb-a3ea7fdb8afc,DISK] 2024-11-19T07:13:25,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-19T07:13:25,244 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/MasterData/WALs/db0afee3eab9,43779,1732000403834/db0afee3eab9%2C43779%2C1732000403834.1732000405174 2024-11-19T07:13:25,245 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32827:32827),(127.0.0.1/127.0.0.1:33965:33965),(127.0.0.1/127.0.0.1:39091:39091)] 2024-11-19T07:13:25,246 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-19T07:13:25,246 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:25,250 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T07:13:25,251 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T07:13:25,286 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T07:13:25,309 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-19T07:13:25,312 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:25,315 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T07:13:25,315 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T07:13:25,319 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-19T07:13:25,319 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:25,320 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:25,320 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T07:13:25,323 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-19T07:13:25,323 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:25,324 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:25,324 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T07:13:25,327 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-19T07:13:25,328 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:25,329 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:25,329 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T07:13:25,334 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T07:13:25,335 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T07:13:25,342 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T07:13:25,342 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T07:13:25,347 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T07:13:25,350 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T07:13:25,355 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T07:13:25,356 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75230381, jitterRate=0.12102003395557404}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T07:13:25,363 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732000405262Initializing all the Stores at 1732000405265 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732000405265Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000405266 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000405266Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000405266Cleaning up temporary data from old regions at 1732000405343 (+77 ms)Region opened successfully at 1732000405363 (+20 ms) 2024-11-19T07:13:25,365 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-19T07:13:25,394 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13a86e70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=db0afee3eab9/172.17.0.2:0 2024-11-19T07:13:25,421 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-19T07:13:25,430 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T07:13:25,430 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T07:13:25,432 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T07:13:25,433 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-19T07:13:25,438 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-19T07:13:25,438 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-19T07:13:25,460 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-19T07:13:25,467 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-19T07:13:25,606 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-19T07:13:25,608 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-19T07:13:25,610 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-19T07:13:25,717 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-19T07:13:25,719 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-19T07:13:25,723 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-19T07:13:25,743 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-19T07:13:25,745 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-19T07:13:25,754 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-19T07:13:25,770 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-19T07:13:25,775 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-19T07:13:25,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T07:13:25,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T07:13:25,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x10151f0c9da0003, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T07:13:25,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T07:13:25,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T07:13:25,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T07:13:25,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x10151f0c9da0003, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T07:13:25,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T07:13:25,789 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=db0afee3eab9,43779,1732000403834, sessionid=0x10151f0c9da0000, setting cluster-up flag (Was=false) 2024-11-19T07:13:25,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x10151f0c9da0003, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T07:13:25,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T07:13:25,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T07:13:25,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T07:13:25,849 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-19T07:13:25,851 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=db0afee3eab9,43779,1732000403834 2024-11-19T07:13:25,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T07:13:25,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T07:13:25,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x10151f0c9da0003, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T07:13:25,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T07:13:25,902 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-19T07:13:25,906 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=db0afee3eab9,43779,1732000403834 2024-11-19T07:13:25,915 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-19T07:13:25,978 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T07:13:25,986 INFO [RS:2;db0afee3eab9:37853 {}] regionserver.HRegionServer(746): ClusterId : 1a1c4a15-1b2b-4e90-9845-b6cc07547cfd 2024-11-19T07:13:25,986 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.HRegionServer(746): ClusterId : 1a1c4a15-1b2b-4e90-9845-b6cc07547cfd 2024-11-19T07:13:25,986 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.HRegionServer(746): ClusterId : 1a1c4a15-1b2b-4e90-9845-b6cc07547cfd 2024-11-19T07:13:25,987 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-19T07:13:25,989 DEBUG [RS:1;db0afee3eab9:42435 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T07:13:25,989 DEBUG [RS:0;db0afee3eab9:34425 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T07:13:25,989 DEBUG [RS:2;db0afee3eab9:37853 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T07:13:25,993 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-19T07:13:25,999 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: db0afee3eab9,43779,1732000403834 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-19T07:13:26,008 DEBUG [RS:1;db0afee3eab9:42435 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T07:13:26,008 DEBUG [RS:2;db0afee3eab9:37853 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T07:13:26,008 DEBUG [RS:0;db0afee3eab9:34425 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T07:13:26,009 DEBUG [RS:1;db0afee3eab9:42435 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T07:13:26,009 DEBUG [RS:2;db0afee3eab9:37853 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T07:13:26,009 DEBUG [RS:0;db0afee3eab9:34425 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T07:13:26,010 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5 2024-11-19T07:13:26,010 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5 2024-11-19T07:13:26,010 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5 2024-11-19T07:13:26,010 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5 2024-11-19T07:13:26,010 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/db0afee3eab9:0, corePoolSize=10, maxPoolSize=10 2024-11-19T07:13:26,011 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,011 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/db0afee3eab9:0, corePoolSize=2, maxPoolSize=2 2024-11-19T07:13:26,011 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,012 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732000436012 2024-11-19T07:13:26,014 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-19T07:13:26,015 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-19T07:13:26,016 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T07:13:26,016 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-19T07:13:26,019 DEBUG [RS:1;db0afee3eab9:42435 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T07:13:26,019 DEBUG [RS:0;db0afee3eab9:34425 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T07:13:26,019 DEBUG [RS:2;db0afee3eab9:37853 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T07:13:26,019 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-19T07:13:26,019 DEBUG [RS:2;db0afee3eab9:37853 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d936a91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=db0afee3eab9/172.17.0.2:0 2024-11-19T07:13:26,019 DEBUG [RS:1;db0afee3eab9:42435 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750b30fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=db0afee3eab9/172.17.0.2:0 2024-11-19T07:13:26,019 DEBUG [RS:0;db0afee3eab9:34425 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3855774c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=db0afee3eab9/172.17.0.2:0 2024-11-19T07:13:26,020 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-19T07:13:26,020 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-19T07:13:26,020 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-19T07:13:26,021 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,023 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:26,023 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-19T07:13:26,025 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-19T07:13:26,026 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-19T07:13:26,027 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-19T07:13:26,032 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-19T07:13:26,033 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-19T07:13:26,035 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.large.0-1732000406034,5,FailOnTimeoutGroup] 2024-11-19T07:13:26,036 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.small.0-1732000406035,5,FailOnTimeoutGroup] 2024-11-19T07:13:26,036 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,036 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-19T07:13:26,037 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,037 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,039 DEBUG [RS:0;db0afee3eab9:34425 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;db0afee3eab9:34425 2024-11-19T07:13:26,039 DEBUG [RS:1;db0afee3eab9:42435 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;db0afee3eab9:42435 2024-11-19T07:13:26,039 DEBUG [RS:2;db0afee3eab9:37853 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;db0afee3eab9:37853 2024-11-19T07:13:26,043 INFO [RS:2;db0afee3eab9:37853 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T07:13:26,043 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T07:13:26,043 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T07:13:26,043 INFO [RS:2;db0afee3eab9:37853 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T07:13:26,043 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T07:13:26,043 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T07:13:26,043 DEBUG [RS:2;db0afee3eab9:37853 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T07:13:26,043 DEBUG [RS:0;db0afee3eab9:34425 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T07:13:26,043 DEBUG [RS:1;db0afee3eab9:42435 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T07:13:26,047 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.HRegionServer(2659): reportForDuty to master=db0afee3eab9,43779,1732000403834 with port=34425, startcode=1732000404604 2024-11-19T07:13:26,047 INFO [RS:2;db0afee3eab9:37853 {}] regionserver.HRegionServer(2659): reportForDuty to master=db0afee3eab9,43779,1732000403834 with port=37853, startcode=1732000404756 2024-11-19T07:13:26,047 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.HRegionServer(2659): reportForDuty to master=db0afee3eab9,43779,1732000403834 with port=42435, startcode=1732000404708 2024-11-19T07:13:26,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741831_1007 (size=1321) 2024-11-19T07:13:26,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741831_1007 (size=1321) 2024-11-19T07:13:26,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741831_1007 (size=1321) 2024-11-19T07:13:26,051 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-19T07:13:26,052 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:26,062 DEBUG [RS:0;db0afee3eab9:34425 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T07:13:26,062 DEBUG [RS:1;db0afee3eab9:42435 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T07:13:26,062 DEBUG [RS:2;db0afee3eab9:37853 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T07:13:26,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741832_1008 (size=32) 2024-11-19T07:13:26,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741832_1008 (size=32) 2024-11-19T07:13:26,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741832_1008 (size=32) 2024-11-19T07:13:26,076 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:26,080 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T07:13:26,085 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T07:13:26,085 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:26,088 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T07:13:26,090 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T07:13:26,095 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T07:13:26,095 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:26,097 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T07:13:26,097 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T07:13:26,102 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T07:13:26,102 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:26,103 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T07:13:26,104 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T07:13:26,105 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57807, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T07:13:26,105 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42977, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T07:13:26,107 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36599, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T07:13:26,107 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T07:13:26,107 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:26,109 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T07:13:26,109 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T07:13:26,111 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740 2024-11-19T07:13:26,113 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740 2024-11-19T07:13:26,113 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] master.ServerManager(363): Checking decommissioned status of RegionServer db0afee3eab9,42435,1732000404708 2024-11-19T07:13:26,116 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] master.ServerManager(517): Registering regionserver=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:26,116 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T07:13:26,116 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T07:13:26,117 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T07:13:26,122 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T07:13:26,129 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] master.ServerManager(363): Checking decommissioned status of RegionServer db0afee3eab9,34425,1732000404604 2024-11-19T07:13:26,129 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T07:13:26,130 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] master.ServerManager(517): Registering regionserver=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:26,131 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67244709, jitterRate=0.002024248242378235}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T07:13:26,134 DEBUG [RS:1;db0afee3eab9:42435 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:26,135 DEBUG [RS:1;db0afee3eab9:42435 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38229 2024-11-19T07:13:26,135 DEBUG [RS:1;db0afee3eab9:42435 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T07:13:26,136 DEBUG [RS:0;db0afee3eab9:34425 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:26,136 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] master.ServerManager(363): Checking decommissioned status of RegionServer db0afee3eab9,37853,1732000404756 2024-11-19T07:13:26,136 DEBUG [RS:0;db0afee3eab9:34425 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38229 2024-11-19T07:13:26,136 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] master.ServerManager(517): Registering regionserver=db0afee3eab9,37853,1732000404756 2024-11-19T07:13:26,137 DEBUG [RS:0;db0afee3eab9:34425 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T07:13:26,137 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732000406076Initializing all the Stores at 1732000406079 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732000406079Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732000406080 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000406080Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732000406080Cleaning up temporary data from old regions at 1732000406116 (+36 ms)Region opened successfully at 1732000406137 (+21 ms) 2024-11-19T07:13:26,137 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T07:13:26,138 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T07:13:26,138 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T07:13:26,138 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T07:13:26,138 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T07:13:26,141 DEBUG [RS:2;db0afee3eab9:37853 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:26,141 DEBUG [RS:2;db0afee3eab9:37853 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38229 2024-11-19T07:13:26,141 DEBUG [RS:2;db0afee3eab9:37853 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T07:13:26,145 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T07:13:26,145 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732000406137Disabling compacts and flushes for region at 1732000406137Disabling writes for close at 1732000406138 (+1 ms)Writing region close event to WAL at 1732000406144 (+6 ms)Closed at 1732000406145 (+1 ms) 2024-11-19T07:13:26,148 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T07:13:26,148 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-19T07:13:26,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-19T07:13:26,168 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T07:13:26,172 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-19T07:13:26,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T07:13:26,218 DEBUG [RS:1;db0afee3eab9:42435 {}] zookeeper.ZKUtil(111): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/db0afee3eab9,42435,1732000404708 2024-11-19T07:13:26,218 WARN [RS:1;db0afee3eab9:42435 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T07:13:26,218 DEBUG [RS:0;db0afee3eab9:34425 {}] zookeeper.ZKUtil(111): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/db0afee3eab9,34425,1732000404604 2024-11-19T07:13:26,218 WARN [RS:0;db0afee3eab9:34425 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T07:13:26,218 DEBUG [RS:2;db0afee3eab9:37853 {}] zookeeper.ZKUtil(111): regionserver:37853-0x10151f0c9da0003, quorum=127.0.0.1:53655, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/db0afee3eab9,37853,1732000404756 2024-11-19T07:13:26,218 INFO [RS:1;db0afee3eab9:42435 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T07:13:26,218 INFO [RS:0;db0afee3eab9:34425 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T07:13:26,218 WARN [RS:2;db0afee3eab9:37853 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T07:13:26,218 INFO [RS:2;db0afee3eab9:37853 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T07:13:26,218 DEBUG [RS:0;db0afee3eab9:34425 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,34425,1732000404604 2024-11-19T07:13:26,218 DEBUG [RS:1;db0afee3eab9:42435 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,42435,1732000404708 2024-11-19T07:13:26,219 DEBUG [RS:2;db0afee3eab9:37853 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756 2024-11-19T07:13:26,220 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [db0afee3eab9,34425,1732000404604] 2024-11-19T07:13:26,220 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [db0afee3eab9,37853,1732000404756] 2024-11-19T07:13:26,220 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [db0afee3eab9,42435,1732000404708] 2024-11-19T07:13:26,244 INFO [RS:2;db0afee3eab9:37853 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T07:13:26,244 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T07:13:26,244 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T07:13:26,262 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T07:13:26,262 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T07:13:26,262 INFO [RS:2;db0afee3eab9:37853 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T07:13:26,268 INFO [RS:2;db0afee3eab9:37853 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T07:13:26,268 INFO [RS:1;db0afee3eab9:42435 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T07:13:26,268 INFO [RS:0;db0afee3eab9:34425 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T07:13:26,268 INFO [RS:2;db0afee3eab9:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,268 INFO [RS:0;db0afee3eab9:34425 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,268 INFO [RS:1;db0afee3eab9:42435 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,273 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T07:13:26,273 INFO [RS:2;db0afee3eab9:37853 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T07:13:26,273 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T07:13:26,279 INFO [RS:0;db0afee3eab9:34425 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T07:13:26,279 INFO [RS:2;db0afee3eab9:37853 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T07:13:26,279 INFO [RS:1;db0afee3eab9:42435 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T07:13:26,280 INFO [RS:2;db0afee3eab9:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,280 INFO [RS:0;db0afee3eab9:34425 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,280 INFO [RS:1;db0afee3eab9:42435 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,281 DEBUG [RS:2;db0afee3eab9:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,281 DEBUG [RS:1;db0afee3eab9:42435 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,281 DEBUG [RS:0;db0afee3eab9:34425 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,281 DEBUG [RS:0;db0afee3eab9:34425 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,281 DEBUG [RS:2;db0afee3eab9:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,281 DEBUG [RS:1;db0afee3eab9:42435 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,281 DEBUG [RS:0;db0afee3eab9:34425 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,281 DEBUG [RS:2;db0afee3eab9:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,281 DEBUG [RS:1;db0afee3eab9:42435 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,281 DEBUG [RS:0;db0afee3eab9:34425 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,281 DEBUG [RS:1;db0afee3eab9:42435 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,281 DEBUG [RS:2;db0afee3eab9:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,281 DEBUG [RS:0;db0afee3eab9:34425 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,281 DEBUG [RS:2;db0afee3eab9:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,281 DEBUG [RS:1;db0afee3eab9:42435 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,282 DEBUG [RS:0;db0afee3eab9:34425 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0, corePoolSize=2, maxPoolSize=2 2024-11-19T07:13:26,282 DEBUG [RS:2;db0afee3eab9:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0, corePoolSize=2, maxPoolSize=2 2024-11-19T07:13:26,282 DEBUG [RS:1;db0afee3eab9:42435 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0, corePoolSize=2, maxPoolSize=2 2024-11-19T07:13:26,282 DEBUG [RS:0;db0afee3eab9:34425 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,282 DEBUG [RS:1;db0afee3eab9:42435 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,282 DEBUG [RS:2;db0afee3eab9:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,282 DEBUG [RS:0;db0afee3eab9:34425 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,282 DEBUG [RS:1;db0afee3eab9:42435 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,282 DEBUG [RS:0;db0afee3eab9:34425 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,282 DEBUG [RS:2;db0afee3eab9:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,282 DEBUG [RS:1;db0afee3eab9:42435 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,282 DEBUG [RS:0;db0afee3eab9:34425 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,282 DEBUG [RS:2;db0afee3eab9:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,282 DEBUG [RS:0;db0afee3eab9:34425 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,282 DEBUG [RS:1;db0afee3eab9:42435 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,282 DEBUG [RS:2;db0afee3eab9:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,282 DEBUG [RS:0;db0afee3eab9:34425 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,282 DEBUG [RS:1;db0afee3eab9:42435 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,282 DEBUG [RS:2;db0afee3eab9:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,282 DEBUG [RS:0;db0afee3eab9:34425 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/db0afee3eab9:0, corePoolSize=3, maxPoolSize=3 2024-11-19T07:13:26,283 DEBUG [RS:1;db0afee3eab9:42435 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,283 DEBUG [RS:2;db0afee3eab9:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1 2024-11-19T07:13:26,283 DEBUG [RS:0;db0afee3eab9:34425 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0, corePoolSize=3, maxPoolSize=3 2024-11-19T07:13:26,283 DEBUG [RS:1;db0afee3eab9:42435 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/db0afee3eab9:0, corePoolSize=3, maxPoolSize=3 2024-11-19T07:13:26,283 DEBUG [RS:2;db0afee3eab9:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/db0afee3eab9:0, corePoolSize=3, maxPoolSize=3 2024-11-19T07:13:26,283 DEBUG [RS:1;db0afee3eab9:42435 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0, corePoolSize=3, maxPoolSize=3 2024-11-19T07:13:26,283 DEBUG [RS:2;db0afee3eab9:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0, corePoolSize=3, maxPoolSize=3 2024-11-19T07:13:26,290 INFO [RS:0;db0afee3eab9:34425 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,290 INFO [RS:1;db0afee3eab9:42435 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,290 INFO [RS:0;db0afee3eab9:34425 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,290 INFO [RS:1;db0afee3eab9:42435 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,290 INFO [RS:0;db0afee3eab9:34425 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,290 INFO [RS:1;db0afee3eab9:42435 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,290 INFO [RS:0;db0afee3eab9:34425 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,290 INFO [RS:1;db0afee3eab9:42435 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,290 INFO [RS:0;db0afee3eab9:34425 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,290 INFO [RS:1;db0afee3eab9:42435 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,290 INFO [RS:1;db0afee3eab9:42435 {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,42435,1732000404708-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T07:13:26,290 INFO [RS:0;db0afee3eab9:34425 {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,34425,1732000404604-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T07:13:26,296 INFO [RS:2;db0afee3eab9:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,297 INFO [RS:2;db0afee3eab9:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,298 INFO [RS:2;db0afee3eab9:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,298 INFO [RS:2;db0afee3eab9:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,298 INFO [RS:2;db0afee3eab9:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,298 INFO [RS:2;db0afee3eab9:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,37853,1732000404756-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T07:13:26,312 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T07:13:26,315 INFO [RS:1;db0afee3eab9:42435 {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,42435,1732000404708-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,315 INFO [RS:1;db0afee3eab9:42435 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,316 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.Replication(171): db0afee3eab9,42435,1732000404708 started 2024-11-19T07:13:26,319 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T07:13:26,320 INFO [RS:0;db0afee3eab9:34425 {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,34425,1732000404604-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,320 INFO [RS:0;db0afee3eab9:34425 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,320 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.Replication(171): db0afee3eab9,34425,1732000404604 started 2024-11-19T07:13:26,322 INFO [RS:2;db0afee3eab9:37853 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T07:13:26,322 INFO [RS:2;db0afee3eab9:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,37853,1732000404756-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,323 INFO [RS:2;db0afee3eab9:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,323 INFO [RS:2;db0afee3eab9:37853 {}] regionserver.Replication(171): db0afee3eab9,37853,1732000404756 started 2024-11-19T07:13:26,323 WARN [db0afee3eab9:43779 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-19T07:13:26,337 INFO [RS:1;db0afee3eab9:42435 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,338 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.HRegionServer(1482): Serving as db0afee3eab9,42435,1732000404708, RpcServer on db0afee3eab9/172.17.0.2:42435, sessionid=0x10151f0c9da0002 2024-11-19T07:13:26,339 DEBUG [RS:1;db0afee3eab9:42435 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T07:13:26,339 DEBUG [RS:1;db0afee3eab9:42435 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager db0afee3eab9,42435,1732000404708 2024-11-19T07:13:26,339 DEBUG [RS:1;db0afee3eab9:42435 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'db0afee3eab9,42435,1732000404708' 2024-11-19T07:13:26,340 DEBUG [RS:1;db0afee3eab9:42435 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T07:13:26,341 DEBUG [RS:1;db0afee3eab9:42435 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T07:13:26,341 DEBUG [RS:1;db0afee3eab9:42435 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T07:13:26,341 DEBUG [RS:1;db0afee3eab9:42435 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T07:13:26,342 DEBUG [RS:1;db0afee3eab9:42435 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager db0afee3eab9,42435,1732000404708 2024-11-19T07:13:26,342 DEBUG [RS:1;db0afee3eab9:42435 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'db0afee3eab9,42435,1732000404708' 2024-11-19T07:13:26,342 DEBUG [RS:1;db0afee3eab9:42435 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T07:13:26,343 DEBUG [RS:1;db0afee3eab9:42435 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T07:13:26,343 DEBUG [RS:1;db0afee3eab9:42435 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T07:13:26,344 INFO [RS:1;db0afee3eab9:42435 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T07:13:26,344 INFO [RS:0;db0afee3eab9:34425 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,344 INFO [RS:1;db0afee3eab9:42435 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T07:13:26,344 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.HRegionServer(1482): Serving as db0afee3eab9,34425,1732000404604, RpcServer on db0afee3eab9/172.17.0.2:34425, sessionid=0x10151f0c9da0001 2024-11-19T07:13:26,344 DEBUG [RS:0;db0afee3eab9:34425 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T07:13:26,344 DEBUG [RS:0;db0afee3eab9:34425 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager db0afee3eab9,34425,1732000404604 2024-11-19T07:13:26,344 DEBUG [RS:0;db0afee3eab9:34425 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'db0afee3eab9,34425,1732000404604' 2024-11-19T07:13:26,344 DEBUG [RS:0;db0afee3eab9:34425 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T07:13:26,345 DEBUG [RS:0;db0afee3eab9:34425 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T07:13:26,346 INFO [RS:2;db0afee3eab9:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:26,346 DEBUG [RS:0;db0afee3eab9:34425 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T07:13:26,346 DEBUG [RS:0;db0afee3eab9:34425 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T07:13:26,346 INFO [RS:2;db0afee3eab9:37853 {}] regionserver.HRegionServer(1482): Serving as db0afee3eab9,37853,1732000404756, RpcServer on db0afee3eab9/172.17.0.2:37853, sessionid=0x10151f0c9da0003 2024-11-19T07:13:26,346 DEBUG [RS:0;db0afee3eab9:34425 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager db0afee3eab9,34425,1732000404604 2024-11-19T07:13:26,346 DEBUG [RS:0;db0afee3eab9:34425 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'db0afee3eab9,34425,1732000404604' 2024-11-19T07:13:26,346 DEBUG [RS:0;db0afee3eab9:34425 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T07:13:26,346 DEBUG [RS:2;db0afee3eab9:37853 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T07:13:26,346 DEBUG [RS:2;db0afee3eab9:37853 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager db0afee3eab9,37853,1732000404756 2024-11-19T07:13:26,346 DEBUG [RS:2;db0afee3eab9:37853 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'db0afee3eab9,37853,1732000404756' 2024-11-19T07:13:26,346 DEBUG [RS:2;db0afee3eab9:37853 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T07:13:26,347 DEBUG [RS:0;db0afee3eab9:34425 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T07:13:26,347 DEBUG [RS:2;db0afee3eab9:37853 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T07:13:26,347 DEBUG [RS:0;db0afee3eab9:34425 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T07:13:26,347 INFO [RS:0;db0afee3eab9:34425 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T07:13:26,347 INFO [RS:0;db0afee3eab9:34425 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T07:13:26,347 DEBUG [RS:2;db0afee3eab9:37853 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T07:13:26,348 DEBUG [RS:2;db0afee3eab9:37853 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T07:13:26,348 DEBUG [RS:2;db0afee3eab9:37853 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager db0afee3eab9,37853,1732000404756 2024-11-19T07:13:26,348 DEBUG [RS:2;db0afee3eab9:37853 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'db0afee3eab9,37853,1732000404756' 2024-11-19T07:13:26,348 DEBUG [RS:2;db0afee3eab9:37853 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T07:13:26,348 DEBUG [RS:2;db0afee3eab9:37853 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T07:13:26,349 DEBUG [RS:2;db0afee3eab9:37853 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T07:13:26,349 INFO [RS:2;db0afee3eab9:37853 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T07:13:26,349 INFO [RS:2;db0afee3eab9:37853 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T07:13:26,450 INFO [RS:1;db0afee3eab9:42435 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T07:13:26,450 INFO [RS:0;db0afee3eab9:34425 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T07:13:26,450 INFO [RS:2;db0afee3eab9:37853 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T07:13:26,453 INFO [RS:1;db0afee3eab9:42435 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db0afee3eab9%2C42435%2C1732000404708, suffix=, logDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,42435,1732000404708, archiveDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/oldWALs, maxLogs=32 2024-11-19T07:13:26,453 INFO [RS:0;db0afee3eab9:34425 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db0afee3eab9%2C34425%2C1732000404604, suffix=, logDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,34425,1732000404604, archiveDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/oldWALs, maxLogs=32 2024-11-19T07:13:26,454 INFO [RS:2;db0afee3eab9:37853 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db0afee3eab9%2C37853%2C1732000404756, suffix=, logDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756, archiveDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/oldWALs, maxLogs=32 2024-11-19T07:13:26,469 DEBUG [RS:1;db0afee3eab9:42435 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,42435,1732000404708/db0afee3eab9%2C42435%2C1732000404708.1732000406456, exclude list is [], retry=0 2024-11-19T07:13:26,469 DEBUG [RS:0;db0afee3eab9:34425 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,34425,1732000404604/db0afee3eab9%2C34425%2C1732000404604.1732000406456, exclude list is [], retry=0 2024-11-19T07:13:26,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41661,DS-e0056ec2-0746-4cc1-bce1-20ef1e1aeac2,DISK] 2024-11-19T07:13:26,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41661,DS-e0056ec2-0746-4cc1-bce1-20ef1e1aeac2,DISK] 2024-11-19T07:13:26,493 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44773,DS-671bf5ab-8128-4e2b-addb-a3ea7fdb8afc,DISK] 2024-11-19T07:13:26,493 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39225,DS-78acb0be-8b81-4317-958b-6530bbb93025,DISK] 2024-11-19T07:13:26,493 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44773,DS-671bf5ab-8128-4e2b-addb-a3ea7fdb8afc,DISK] 2024-11-19T07:13:26,493 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39225,DS-78acb0be-8b81-4317-958b-6530bbb93025,DISK] 2024-11-19T07:13:26,497 INFO [RS:1;db0afee3eab9:42435 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,42435,1732000404708/db0afee3eab9%2C42435%2C1732000404708.1732000406456 2024-11-19T07:13:26,497 INFO [RS:0;db0afee3eab9:34425 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,34425,1732000404604/db0afee3eab9%2C34425%2C1732000404604.1732000406456 2024-11-19T07:13:26,500 DEBUG [RS:1;db0afee3eab9:42435 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39091:39091),(127.0.0.1/127.0.0.1:33965:33965),(127.0.0.1/127.0.0.1:32827:32827)] 2024-11-19T07:13:26,503 DEBUG [RS:2;db0afee3eab9:37853 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756/db0afee3eab9%2C37853%2C1732000404756.1732000406457, exclude list is [], retry=0 2024-11-19T07:13:26,504 DEBUG [RS:0;db0afee3eab9:34425 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39091:39091),(127.0.0.1/127.0.0.1:32827:32827),(127.0.0.1/127.0.0.1:33965:33965)] 2024-11-19T07:13:26,509 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44773,DS-671bf5ab-8128-4e2b-addb-a3ea7fdb8afc,DISK] 2024-11-19T07:13:26,509 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41661,DS-e0056ec2-0746-4cc1-bce1-20ef1e1aeac2,DISK] 2024-11-19T07:13:26,509 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39225,DS-78acb0be-8b81-4317-958b-6530bbb93025,DISK] 2024-11-19T07:13:26,516 INFO [RS:2;db0afee3eab9:37853 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756/db0afee3eab9%2C37853%2C1732000404756.1732000406457 2024-11-19T07:13:26,516 DEBUG [RS:2;db0afee3eab9:37853 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33965:33965),(127.0.0.1/127.0.0.1:39091:39091),(127.0.0.1/127.0.0.1:32827:32827)] 2024-11-19T07:13:26,575 DEBUG [db0afee3eab9:43779 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-19T07:13:26,582 DEBUG [db0afee3eab9:43779 {}] balancer.BalancerClusterState(204): Hosts are {db0afee3eab9=0} racks are {/default-rack=0} 2024-11-19T07:13:26,589 DEBUG [db0afee3eab9:43779 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-19T07:13:26,589 DEBUG [db0afee3eab9:43779 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-19T07:13:26,589 DEBUG [db0afee3eab9:43779 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-19T07:13:26,589 DEBUG [db0afee3eab9:43779 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-19T07:13:26,589 DEBUG [db0afee3eab9:43779 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-19T07:13:26,590 DEBUG [db0afee3eab9:43779 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-19T07:13:26,590 INFO [db0afee3eab9:43779 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-19T07:13:26,590 INFO [db0afee3eab9:43779 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-19T07:13:26,590 INFO [db0afee3eab9:43779 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-19T07:13:26,590 DEBUG [db0afee3eab9:43779 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-19T07:13:26,596 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=db0afee3eab9,37853,1732000404756 2024-11-19T07:13:26,603 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,37853,1732000404756, state=OPENING 2024-11-19T07:13:26,649 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-19T07:13:26,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x10151f0c9da0003, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T07:13:26,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T07:13:26,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T07:13:26,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T07:13:26,660 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:26,660 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:26,661 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:26,661 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:26,662 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T07:13:26,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,37853,1732000404756}] 2024-11-19T07:13:26,843 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T07:13:26,844 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60849, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T07:13:26,857 INFO [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-19T07:13:26,857 INFO [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T07:13:26,858 INFO [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-19T07:13:26,861 INFO [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db0afee3eab9%2C37853%2C1732000404756.meta, suffix=.meta, logDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756, archiveDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/oldWALs, maxLogs=32 2024-11-19T07:13:26,877 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756/db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, exclude list is [], retry=0 2024-11-19T07:13:26,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41661,DS-e0056ec2-0746-4cc1-bce1-20ef1e1aeac2,DISK] 2024-11-19T07:13:26,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39225,DS-78acb0be-8b81-4317-958b-6530bbb93025,DISK] 2024-11-19T07:13:26,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44773,DS-671bf5ab-8128-4e2b-addb-a3ea7fdb8afc,DISK] 2024-11-19T07:13:26,884 INFO [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756/db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta 2024-11-19T07:13:26,887 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39091:39091),(127.0.0.1/127.0.0.1:32827:32827),(127.0.0.1/127.0.0.1:33965:33965)] 2024-11-19T07:13:26,887 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-19T07:13:26,888 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-19T07:13:26,891 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-19T07:13:26,895 INFO [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-19T07:13:26,899 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-19T07:13:26,900 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:26,900 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-19T07:13:26,900 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-19T07:13:26,904 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T07:13:26,905 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T07:13:26,906 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:26,906 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T07:13:26,907 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T07:13:26,908 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T07:13:26,908 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:26,909 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T07:13:26,909 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T07:13:26,910 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T07:13:26,911 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:26,911 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T07:13:26,911 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T07:13:26,913 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T07:13:26,913 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:26,914 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T07:13:26,914 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T07:13:26,915 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740 2024-11-19T07:13:26,918 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740 2024-11-19T07:13:26,921 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T07:13:26,921 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T07:13:26,922 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T07:13:26,925 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T07:13:26,926 INFO [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62013224, jitterRate=-0.07593095302581787}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T07:13:26,927 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-19T07:13:26,929 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732000406901Writing region info on filesystem at 1732000406901Initializing all the Stores at 1732000406903 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732000406903Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732000406904 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000406904Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732000406904Cleaning up temporary data from old regions at 1732000406921 (+17 ms)Running coprocessor post-open hooks at 1732000406927 (+6 ms)Region opened successfully at 1732000406928 (+1 ms) 2024-11-19T07:13:26,935 INFO [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732000406833 2024-11-19T07:13:26,947 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-19T07:13:26,948 INFO [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-19T07:13:26,950 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,37853,1732000404756 2024-11-19T07:13:26,953 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,37853,1732000404756, state=OPEN 2024-11-19T07:13:27,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:27,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x10151f0c9da0003, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:27,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:27,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:27,007 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:27,007 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:27,007 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:27,007 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:27,008 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=db0afee3eab9,37853,1732000404756 2024-11-19T07:13:27,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-19T07:13:27,014 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,37853,1732000404756 in 345 msec 2024-11-19T07:13:27,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-19T07:13:27,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 862 msec 2024-11-19T07:13:27,023 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T07:13:27,023 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-19T07:13:27,043 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T07:13:27,044 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,37853,1732000404756, seqNum=-1] 2024-11-19T07:13:27,064 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T07:13:27,067 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50777, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T07:13:27,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1410 sec 2024-11-19T07:13:27,086 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732000407086, completionTime=-1 2024-11-19T07:13:27,090 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-19T07:13:27,090 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T07:13:27,122 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-19T07:13:27,122 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732000467122 2024-11-19T07:13:27,122 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732000527122 2024-11-19T07:13:27,122 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 31 msec 2024-11-19T07:13:27,124 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-19T07:13:27,130 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,43779,1732000403834-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:27,130 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,43779,1732000403834-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:27,130 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,43779,1732000403834-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:27,132 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-db0afee3eab9:43779, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:27,132 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:27,133 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:27,140 DEBUG [master/db0afee3eab9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-19T07:13:27,162 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.312sec 2024-11-19T07:13:27,164 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-19T07:13:27,165 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-19T07:13:27,166 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-19T07:13:27,167 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-19T07:13:27,167 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-19T07:13:27,168 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,43779,1732000403834-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T07:13:27,168 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,43779,1732000403834-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-19T07:13:27,173 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-19T07:13:27,174 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-19T07:13:27,175 INFO [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,43779,1732000403834-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T07:13:27,197 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1482df6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T07:13:27,201 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-19T07:13:27,201 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-19T07:13:27,205 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db0afee3eab9,43779,-1 for getting cluster id 2024-11-19T07:13:27,208 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T07:13:27,215 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1a1c4a15-1b2b-4e90-9845-b6cc07547cfd' 2024-11-19T07:13:27,217 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T07:13:27,218 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1a1c4a15-1b2b-4e90-9845-b6cc07547cfd" 2024-11-19T07:13:27,218 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78f024eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T07:13:27,218 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db0afee3eab9,43779,-1] 2024-11-19T07:13:27,220 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T07:13:27,223 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T07:13:27,224 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T07:13:27,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bf9209, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T07:13:27,227 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T07:13:27,234 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,37853,1732000404756, seqNum=-1] 2024-11-19T07:13:27,235 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T07:13:27,237 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T07:13:27,280 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=db0afee3eab9,43779,1732000403834 2024-11-19T07:13:27,285 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T07:13:27,291 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is db0afee3eab9,43779,1732000403834 2024-11-19T07:13:27,294 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@8b2dbd5 2024-11-19T07:13:27,296 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T07:13:27,303 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T07:13:27,309 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T07:13:27,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-19T07:13:27,319 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T07:13:27,322 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-19T07:13:27,322 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:27,325 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T07:13:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T07:13:27,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741837_1013 (size=358) 2024-11-19T07:13:27,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741837_1013 (size=358) 2024-11-19T07:13:27,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741837_1013 (size=358) 2024-11-19T07:13:27,344 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 02309bab61f6207d73b9f19985e1fdb9, NAME => 'test,,1732000407304.02309bab61f6207d73b9f19985e1fdb9.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:27,345 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 7d9d31e1a2abc752a2056ed0e3a3fcec, NAME => 'test,ccc,1732000407304.7d9d31e1a2abc752a2056ed0e3a3fcec.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:27,345 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 12dcc969f79695b5f0d224b605268d2d, NAME => 'test,bbb,1732000407304.12dcc969f79695b5f0d224b605268d2d.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:27,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741839_1015 (size=45) 2024-11-19T07:13:27,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741839_1015 (size=45) 2024-11-19T07:13:27,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741839_1015 (size=45) 2024-11-19T07:13:27,370 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1732000407304.7d9d31e1a2abc752a2056ed0e3a3fcec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:27,370 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 7d9d31e1a2abc752a2056ed0e3a3fcec, disabling compactions & flushes 2024-11-19T07:13:27,370 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1732000407304.7d9d31e1a2abc752a2056ed0e3a3fcec. 2024-11-19T07:13:27,370 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732000407304.7d9d31e1a2abc752a2056ed0e3a3fcec. 2024-11-19T07:13:27,370 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732000407304.7d9d31e1a2abc752a2056ed0e3a3fcec. after waiting 0 ms 2024-11-19T07:13:27,370 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732000407304.7d9d31e1a2abc752a2056ed0e3a3fcec. 2024-11-19T07:13:27,370 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1732000407304.7d9d31e1a2abc752a2056ed0e3a3fcec. 2024-11-19T07:13:27,370 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 7d9d31e1a2abc752a2056ed0e3a3fcec: Waiting for close lock at 1732000407370Disabling compacts and flushes for region at 1732000407370Disabling writes for close at 1732000407370Writing region close event to WAL at 1732000407370Closed at 1732000407370 2024-11-19T07:13:27,371 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 1ece759217c055d9a295034ce69dd86d, NAME => 'test,ddd,1732000407304.1ece759217c055d9a295034ce69dd86d.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:27,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741838_1014 (size=42) 2024-11-19T07:13:27,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741838_1014 (size=42) 2024-11-19T07:13:27,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741838_1014 (size=42) 2024-11-19T07:13:27,374 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1732000407304.02309bab61f6207d73b9f19985e1fdb9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:27,374 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 02309bab61f6207d73b9f19985e1fdb9, disabling compactions & flushes 2024-11-19T07:13:27,374 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1732000407304.02309bab61f6207d73b9f19985e1fdb9. 2024-11-19T07:13:27,374 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732000407304.02309bab61f6207d73b9f19985e1fdb9. 2024-11-19T07:13:27,374 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1732000407304.02309bab61f6207d73b9f19985e1fdb9. after waiting 0 ms 2024-11-19T07:13:27,374 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1732000407304.02309bab61f6207d73b9f19985e1fdb9. 2024-11-19T07:13:27,374 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1732000407304.02309bab61f6207d73b9f19985e1fdb9. 2024-11-19T07:13:27,374 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 02309bab61f6207d73b9f19985e1fdb9: Waiting for close lock at 1732000407374Disabling compacts and flushes for region at 1732000407374Disabling writes for close at 1732000407374Writing region close event to WAL at 1732000407374Closed at 1732000407374 2024-11-19T07:13:27,375 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e880c03f4c433baa6242dd65ef2d8486, NAME => 'test,eee,1732000407304.e880c03f4c433baa6242dd65ef2d8486.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:27,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741840_1016 (size=45) 2024-11-19T07:13:27,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741840_1016 (size=45) 2024-11-19T07:13:27,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741840_1016 (size=45) 2024-11-19T07:13:27,385 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1732000407304.12dcc969f79695b5f0d224b605268d2d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:27,385 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 12dcc969f79695b5f0d224b605268d2d, disabling compactions & flushes 2024-11-19T07:13:27,385 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1732000407304.12dcc969f79695b5f0d224b605268d2d. 2024-11-19T07:13:27,385 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732000407304.12dcc969f79695b5f0d224b605268d2d. 2024-11-19T07:13:27,385 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732000407304.12dcc969f79695b5f0d224b605268d2d. after waiting 0 ms 2024-11-19T07:13:27,385 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732000407304.12dcc969f79695b5f0d224b605268d2d. 2024-11-19T07:13:27,385 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1732000407304.12dcc969f79695b5f0d224b605268d2d. 2024-11-19T07:13:27,385 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 12dcc969f79695b5f0d224b605268d2d: Waiting for close lock at 1732000407385Disabling compacts and flushes for region at 1732000407385Disabling writes for close at 1732000407385Writing region close event to WAL at 1732000407385Closed at 1732000407385 2024-11-19T07:13:27,386 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 49b850b07eedb76caa94e4c93ac5c832, NAME => 'test,fff,1732000407304.49b850b07eedb76caa94e4c93ac5c832.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:27,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741841_1017 (size=45) 2024-11-19T07:13:27,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741842_1018 (size=45) 2024-11-19T07:13:27,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741841_1017 (size=45) 2024-11-19T07:13:27,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741842_1018 (size=45) 2024-11-19T07:13:27,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741841_1017 (size=45) 2024-11-19T07:13:27,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741842_1018 (size=45) 2024-11-19T07:13:27,405 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,eee,1732000407304.e880c03f4c433baa6242dd65ef2d8486.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:27,405 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ddd,1732000407304.1ece759217c055d9a295034ce69dd86d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:27,405 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing e880c03f4c433baa6242dd65ef2d8486, disabling compactions & flushes 2024-11-19T07:13:27,405 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 1ece759217c055d9a295034ce69dd86d, disabling compactions & flushes 2024-11-19T07:13:27,405 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ddd,1732000407304.1ece759217c055d9a295034ce69dd86d. 2024-11-19T07:13:27,405 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,eee,1732000407304.e880c03f4c433baa6242dd65ef2d8486. 2024-11-19T07:13:27,405 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732000407304.1ece759217c055d9a295034ce69dd86d. 2024-11-19T07:13:27,405 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732000407304.e880c03f4c433baa6242dd65ef2d8486. 2024-11-19T07:13:27,405 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732000407304.1ece759217c055d9a295034ce69dd86d. after waiting 0 ms 2024-11-19T07:13:27,405 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732000407304.e880c03f4c433baa6242dd65ef2d8486. after waiting 0 ms 2024-11-19T07:13:27,405 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732000407304.e880c03f4c433baa6242dd65ef2d8486. 2024-11-19T07:13:27,405 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732000407304.1ece759217c055d9a295034ce69dd86d. 2024-11-19T07:13:27,405 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,eee,1732000407304.e880c03f4c433baa6242dd65ef2d8486. 2024-11-19T07:13:27,405 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ddd,1732000407304.1ece759217c055d9a295034ce69dd86d. 2024-11-19T07:13:27,405 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for e880c03f4c433baa6242dd65ef2d8486: Waiting for close lock at 1732000407405Disabling compacts and flushes for region at 1732000407405Disabling writes for close at 1732000407405Writing region close event to WAL at 1732000407405Closed at 1732000407405 2024-11-19T07:13:27,405 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 1ece759217c055d9a295034ce69dd86d: Waiting for close lock at 1732000407405Disabling compacts and flushes for region at 1732000407405Disabling writes for close at 1732000407405Writing region close event to WAL at 1732000407405Closed at 1732000407405 2024-11-19T07:13:27,406 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 66fe58970f9c4a1cba9c3e3da843ed3d, NAME => 'test,hhh,1732000407304.66fe58970f9c4a1cba9c3e3da843ed3d.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:27,406 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => db4ba820395e8b4a2b38d57f8a222e97, NAME => 'test,ggg,1732000407304.db4ba820395e8b4a2b38d57f8a222e97.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:27,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741843_1019 (size=45) 2024-11-19T07:13:27,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741843_1019 (size=45) 2024-11-19T07:13:27,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741843_1019 (size=45) 2024-11-19T07:13:27,419 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,fff,1732000407304.49b850b07eedb76caa94e4c93ac5c832.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:27,419 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 49b850b07eedb76caa94e4c93ac5c832, disabling compactions & flushes 2024-11-19T07:13:27,419 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,fff,1732000407304.49b850b07eedb76caa94e4c93ac5c832. 2024-11-19T07:13:27,419 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732000407304.49b850b07eedb76caa94e4c93ac5c832. 2024-11-19T07:13:27,419 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732000407304.49b850b07eedb76caa94e4c93ac5c832. after waiting 0 ms 2024-11-19T07:13:27,419 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732000407304.49b850b07eedb76caa94e4c93ac5c832. 2024-11-19T07:13:27,419 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,fff,1732000407304.49b850b07eedb76caa94e4c93ac5c832. 2024-11-19T07:13:27,419 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 49b850b07eedb76caa94e4c93ac5c832: Waiting for close lock at 1732000407419Disabling compacts and flushes for region at 1732000407419Disabling writes for close at 1732000407419Writing region close event to WAL at 1732000407419Closed at 1732000407419 2024-11-19T07:13:27,420 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 43676ec232db06c9bf4af64db1ede4f5, NAME => 'test,iii,1732000407304.43676ec232db06c9bf4af64db1ede4f5.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:27,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741845_1021 (size=45) 2024-11-19T07:13:27,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741844_1020 (size=45) 2024-11-19T07:13:27,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741844_1020 (size=45) 2024-11-19T07:13:27,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741845_1021 (size=45) 2024-11-19T07:13:27,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741845_1021 (size=45) 2024-11-19T07:13:27,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741844_1020 (size=45) 2024-11-19T07:13:27,429 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,hhh,1732000407304.66fe58970f9c4a1cba9c3e3da843ed3d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:27,429 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 66fe58970f9c4a1cba9c3e3da843ed3d, disabling compactions & flushes 2024-11-19T07:13:27,429 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,hhh,1732000407304.66fe58970f9c4a1cba9c3e3da843ed3d. 2024-11-19T07:13:27,430 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732000407304.66fe58970f9c4a1cba9c3e3da843ed3d. 2024-11-19T07:13:27,430 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732000407304.66fe58970f9c4a1cba9c3e3da843ed3d. after waiting 0 ms 2024-11-19T07:13:27,430 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732000407304.66fe58970f9c4a1cba9c3e3da843ed3d. 2024-11-19T07:13:27,430 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ggg,1732000407304.db4ba820395e8b4a2b38d57f8a222e97.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:27,430 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,hhh,1732000407304.66fe58970f9c4a1cba9c3e3da843ed3d. 2024-11-19T07:13:27,430 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 66fe58970f9c4a1cba9c3e3da843ed3d: Waiting for close lock at 1732000407429Disabling compacts and flushes for region at 1732000407429Disabling writes for close at 1732000407430 (+1 ms)Writing region close event to WAL at 1732000407430Closed at 1732000407430 2024-11-19T07:13:27,430 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing db4ba820395e8b4a2b38d57f8a222e97, disabling compactions & flushes 2024-11-19T07:13:27,430 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ggg,1732000407304.db4ba820395e8b4a2b38d57f8a222e97. 2024-11-19T07:13:27,430 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732000407304.db4ba820395e8b4a2b38d57f8a222e97. 2024-11-19T07:13:27,430 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4421c1189c5fdff3a40dcc58728c41ab, NAME => 'test,jjj,1732000407304.4421c1189c5fdff3a40dcc58728c41ab.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:27,430 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732000407304.db4ba820395e8b4a2b38d57f8a222e97. after waiting 0 ms 2024-11-19T07:13:27,430 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732000407304.db4ba820395e8b4a2b38d57f8a222e97. 2024-11-19T07:13:27,430 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ggg,1732000407304.db4ba820395e8b4a2b38d57f8a222e97. 2024-11-19T07:13:27,430 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for db4ba820395e8b4a2b38d57f8a222e97: Waiting for close lock at 1732000407430Disabling compacts and flushes for region at 1732000407430Disabling writes for close at 1732000407430Writing region close event to WAL at 1732000407430Closed at 1732000407430 2024-11-19T07:13:27,431 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => bbb518f4693490310eca052bf28f87fc, NAME => 'test,kkk,1732000407304.bbb518f4693490310eca052bf28f87fc.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:27,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741846_1022 (size=45) 2024-11-19T07:13:27,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741846_1022 (size=45) 2024-11-19T07:13:27,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741846_1022 (size=45) 2024-11-19T07:13:27,439 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,iii,1732000407304.43676ec232db06c9bf4af64db1ede4f5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:27,440 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 43676ec232db06c9bf4af64db1ede4f5, disabling compactions & flushes 2024-11-19T07:13:27,440 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,iii,1732000407304.43676ec232db06c9bf4af64db1ede4f5. 2024-11-19T07:13:27,440 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732000407304.43676ec232db06c9bf4af64db1ede4f5. 2024-11-19T07:13:27,440 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732000407304.43676ec232db06c9bf4af64db1ede4f5. after waiting 0 ms 2024-11-19T07:13:27,440 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732000407304.43676ec232db06c9bf4af64db1ede4f5. 2024-11-19T07:13:27,440 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,iii,1732000407304.43676ec232db06c9bf4af64db1ede4f5. 2024-11-19T07:13:27,440 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 43676ec232db06c9bf4af64db1ede4f5: Waiting for close lock at 1732000407440Disabling compacts and flushes for region at 1732000407440Disabling writes for close at 1732000407440Writing region close event to WAL at 1732000407440Closed at 1732000407440 2024-11-19T07:13:27,441 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 82941673e9ad03e5a037091750c3f315, NAME => 'test,lll,1732000407304.82941673e9ad03e5a037091750c3f315.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:27,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T07:13:27,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741848_1024 (size=45) 2024-11-19T07:13:27,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741847_1023 (size=45) 2024-11-19T07:13:27,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741848_1024 (size=45) 2024-11-19T07:13:27,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741848_1024 (size=45) 2024-11-19T07:13:27,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741847_1023 (size=45) 2024-11-19T07:13:27,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741847_1023 (size=45) 2024-11-19T07:13:27,449 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,kkk,1732000407304.bbb518f4693490310eca052bf28f87fc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:27,450 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing bbb518f4693490310eca052bf28f87fc, disabling compactions & flushes 2024-11-19T07:13:27,450 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,kkk,1732000407304.bbb518f4693490310eca052bf28f87fc. 2024-11-19T07:13:27,450 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732000407304.bbb518f4693490310eca052bf28f87fc. 2024-11-19T07:13:27,450 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732000407304.bbb518f4693490310eca052bf28f87fc. after waiting 0 ms 2024-11-19T07:13:27,450 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732000407304.bbb518f4693490310eca052bf28f87fc. 2024-11-19T07:13:27,450 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,kkk,1732000407304.bbb518f4693490310eca052bf28f87fc. 2024-11-19T07:13:27,450 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for bbb518f4693490310eca052bf28f87fc: Waiting for close lock at 1732000407450Disabling compacts and flushes for region at 1732000407450Disabling writes for close at 1732000407450Writing region close event to WAL at 1732000407450Closed at 1732000407450 2024-11-19T07:13:27,450 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => d2ef4078aa4f5f4b6f4dc7188dd2f1cb, NAME => 'test,mmm,1732000407304.d2ef4078aa4f5f4b6f4dc7188dd2f1cb.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:27,452 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,jjj,1732000407304.4421c1189c5fdff3a40dcc58728c41ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:27,452 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 4421c1189c5fdff3a40dcc58728c41ab, disabling compactions & flushes 2024-11-19T07:13:27,452 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,jjj,1732000407304.4421c1189c5fdff3a40dcc58728c41ab. 2024-11-19T07:13:27,452 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732000407304.4421c1189c5fdff3a40dcc58728c41ab. 2024-11-19T07:13:27,453 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732000407304.4421c1189c5fdff3a40dcc58728c41ab. after waiting 0 ms 2024-11-19T07:13:27,453 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732000407304.4421c1189c5fdff3a40dcc58728c41ab. 2024-11-19T07:13:27,453 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,jjj,1732000407304.4421c1189c5fdff3a40dcc58728c41ab. 2024-11-19T07:13:27,453 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4421c1189c5fdff3a40dcc58728c41ab: Waiting for close lock at 1732000407452Disabling compacts and flushes for region at 1732000407452Disabling writes for close at 1732000407453 (+1 ms)Writing region close event to WAL at 1732000407453Closed at 1732000407453 2024-11-19T07:13:27,454 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 74657be6232ad22aaca548d82491de41, NAME => 'test,nnn,1732000407304.74657be6232ad22aaca548d82491de41.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:27,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741849_1025 (size=45) 2024-11-19T07:13:27,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741849_1025 (size=45) 2024-11-19T07:13:27,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741849_1025 (size=45) 2024-11-19T07:13:27,461 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,lll,1732000407304.82941673e9ad03e5a037091750c3f315.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:27,461 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 82941673e9ad03e5a037091750c3f315, disabling compactions & flushes 2024-11-19T07:13:27,461 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,lll,1732000407304.82941673e9ad03e5a037091750c3f315. 2024-11-19T07:13:27,461 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732000407304.82941673e9ad03e5a037091750c3f315. 2024-11-19T07:13:27,461 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732000407304.82941673e9ad03e5a037091750c3f315. after waiting 0 ms 2024-11-19T07:13:27,461 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732000407304.82941673e9ad03e5a037091750c3f315. 2024-11-19T07:13:27,461 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,lll,1732000407304.82941673e9ad03e5a037091750c3f315. 2024-11-19T07:13:27,461 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 82941673e9ad03e5a037091750c3f315: Waiting for close lock at 1732000407461Disabling compacts and flushes for region at 1732000407461Disabling writes for close at 1732000407461Writing region close event to WAL at 1732000407461Closed at 1732000407461 2024-11-19T07:13:27,462 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => fdf73eccbee4f01f5a86b87f17dcdb8b, NAME => 'test,ooo,1732000407304.fdf73eccbee4f01f5a86b87f17dcdb8b.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:27,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741850_1026 (size=45) 2024-11-19T07:13:27,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741850_1026 (size=45) 2024-11-19T07:13:27,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741850_1026 (size=45) 2024-11-19T07:13:27,468 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,nnn,1732000407304.74657be6232ad22aaca548d82491de41.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:27,468 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 74657be6232ad22aaca548d82491de41, disabling compactions & flushes 2024-11-19T07:13:27,468 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,nnn,1732000407304.74657be6232ad22aaca548d82491de41. 2024-11-19T07:13:27,468 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732000407304.74657be6232ad22aaca548d82491de41. 2024-11-19T07:13:27,468 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732000407304.74657be6232ad22aaca548d82491de41. after waiting 0 ms 2024-11-19T07:13:27,468 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732000407304.74657be6232ad22aaca548d82491de41. 2024-11-19T07:13:27,468 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,nnn,1732000407304.74657be6232ad22aaca548d82491de41. 2024-11-19T07:13:27,469 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 74657be6232ad22aaca548d82491de41: Waiting for close lock at 1732000407468Disabling compacts and flushes for region at 1732000407468Disabling writes for close at 1732000407468Writing region close event to WAL at 1732000407468Closed at 1732000407468 2024-11-19T07:13:27,470 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3a637105eb1d8b75696b3cfabe5d8173, NAME => 'test,ppp,1732000407304.3a637105eb1d8b75696b3cfabe5d8173.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:27,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741851_1027 (size=45) 2024-11-19T07:13:27,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741851_1027 (size=45) 2024-11-19T07:13:27,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741851_1027 (size=45) 2024-11-19T07:13:27,480 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,mmm,1732000407304.d2ef4078aa4f5f4b6f4dc7188dd2f1cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:27,481 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing d2ef4078aa4f5f4b6f4dc7188dd2f1cb, disabling compactions & flushes 2024-11-19T07:13:27,481 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,mmm,1732000407304.d2ef4078aa4f5f4b6f4dc7188dd2f1cb. 2024-11-19T07:13:27,481 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732000407304.d2ef4078aa4f5f4b6f4dc7188dd2f1cb. 2024-11-19T07:13:27,481 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732000407304.d2ef4078aa4f5f4b6f4dc7188dd2f1cb. after waiting 0 ms 2024-11-19T07:13:27,481 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732000407304.d2ef4078aa4f5f4b6f4dc7188dd2f1cb. 2024-11-19T07:13:27,481 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,mmm,1732000407304.d2ef4078aa4f5f4b6f4dc7188dd2f1cb. 2024-11-19T07:13:27,481 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for d2ef4078aa4f5f4b6f4dc7188dd2f1cb: Waiting for close lock at 1732000407481Disabling compacts and flushes for region at 1732000407481Disabling writes for close at 1732000407481Writing region close event to WAL at 1732000407481Closed at 1732000407481 2024-11-19T07:13:27,482 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 2df93dc30d9c55d765ada1f50e320789, NAME => 'test,qqq,1732000407304.2df93dc30d9c55d765ada1f50e320789.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:27,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741852_1028 (size=45) 2024-11-19T07:13:27,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741852_1028 (size=45) 2024-11-19T07:13:27,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741852_1028 (size=45) 2024-11-19T07:13:27,494 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ooo,1732000407304.fdf73eccbee4f01f5a86b87f17dcdb8b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:27,494 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing fdf73eccbee4f01f5a86b87f17dcdb8b, disabling compactions & flushes 2024-11-19T07:13:27,494 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ooo,1732000407304.fdf73eccbee4f01f5a86b87f17dcdb8b. 2024-11-19T07:13:27,494 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732000407304.fdf73eccbee4f01f5a86b87f17dcdb8b. 2024-11-19T07:13:27,494 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732000407304.fdf73eccbee4f01f5a86b87f17dcdb8b. after waiting 0 ms 2024-11-19T07:13:27,494 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732000407304.fdf73eccbee4f01f5a86b87f17dcdb8b. 2024-11-19T07:13:27,494 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ooo,1732000407304.fdf73eccbee4f01f5a86b87f17dcdb8b. 2024-11-19T07:13:27,495 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for fdf73eccbee4f01f5a86b87f17dcdb8b: Waiting for close lock at 1732000407494Disabling compacts and flushes for region at 1732000407494Disabling writes for close at 1732000407494Writing region close event to WAL at 1732000407494Closed at 1732000407494 2024-11-19T07:13:27,495 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => dba97405d3b35647d42d266df5e81cec, NAME => 'test,rrr,1732000407304.dba97405d3b35647d42d266df5e81cec.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:27,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741854_1030 (size=45) 2024-11-19T07:13:27,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741854_1030 (size=45) 2024-11-19T07:13:27,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741854_1030 (size=45) 2024-11-19T07:13:27,502 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,qqq,1732000407304.2df93dc30d9c55d765ada1f50e320789.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:27,503 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 2df93dc30d9c55d765ada1f50e320789, disabling compactions & flushes 2024-11-19T07:13:27,503 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,qqq,1732000407304.2df93dc30d9c55d765ada1f50e320789. 2024-11-19T07:13:27,503 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732000407304.2df93dc30d9c55d765ada1f50e320789. 2024-11-19T07:13:27,503 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732000407304.2df93dc30d9c55d765ada1f50e320789. after waiting 0 ms 2024-11-19T07:13:27,503 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732000407304.2df93dc30d9c55d765ada1f50e320789. 2024-11-19T07:13:27,503 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,qqq,1732000407304.2df93dc30d9c55d765ada1f50e320789. 2024-11-19T07:13:27,503 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 2df93dc30d9c55d765ada1f50e320789: Waiting for close lock at 1732000407503Disabling compacts and flushes for region at 1732000407503Disabling writes for close at 1732000407503Writing region close event to WAL at 1732000407503Closed at 1732000407503 2024-11-19T07:13:27,504 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => d703151d82f3a429f348967988283542, NAME => 'test,sss,1732000407304.d703151d82f3a429f348967988283542.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:27,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741853_1029 (size=45) 2024-11-19T07:13:27,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741853_1029 (size=45) 2024-11-19T07:13:27,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741853_1029 (size=45) 2024-11-19T07:13:27,509 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ppp,1732000407304.3a637105eb1d8b75696b3cfabe5d8173.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:27,509 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 3a637105eb1d8b75696b3cfabe5d8173, disabling compactions & flushes 2024-11-19T07:13:27,509 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ppp,1732000407304.3a637105eb1d8b75696b3cfabe5d8173. 2024-11-19T07:13:27,509 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732000407304.3a637105eb1d8b75696b3cfabe5d8173. 2024-11-19T07:13:27,509 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732000407304.3a637105eb1d8b75696b3cfabe5d8173. after waiting 0 ms 2024-11-19T07:13:27,510 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732000407304.3a637105eb1d8b75696b3cfabe5d8173. 2024-11-19T07:13:27,510 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ppp,1732000407304.3a637105eb1d8b75696b3cfabe5d8173. 2024-11-19T07:13:27,510 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3a637105eb1d8b75696b3cfabe5d8173: Waiting for close lock at 1732000407509Disabling compacts and flushes for region at 1732000407509Disabling writes for close at 1732000407510 (+1 ms)Writing region close event to WAL at 1732000407510Closed at 1732000407510 2024-11-19T07:13:27,510 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2f38687f659a16d9e2a781f2a75d514a, NAME => 'test,ttt,1732000407304.2f38687f659a16d9e2a781f2a75d514a.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:27,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741855_1031 (size=45) 2024-11-19T07:13:27,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741855_1031 (size=45) 2024-11-19T07:13:27,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741855_1031 (size=45) 2024-11-19T07:13:27,523 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,rrr,1732000407304.dba97405d3b35647d42d266df5e81cec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:27,523 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing dba97405d3b35647d42d266df5e81cec, disabling compactions & flushes 2024-11-19T07:13:27,523 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,rrr,1732000407304.dba97405d3b35647d42d266df5e81cec. 2024-11-19T07:13:27,523 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732000407304.dba97405d3b35647d42d266df5e81cec. 2024-11-19T07:13:27,523 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732000407304.dba97405d3b35647d42d266df5e81cec. after waiting 0 ms 2024-11-19T07:13:27,524 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732000407304.dba97405d3b35647d42d266df5e81cec. 2024-11-19T07:13:27,524 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,rrr,1732000407304.dba97405d3b35647d42d266df5e81cec. 2024-11-19T07:13:27,524 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for dba97405d3b35647d42d266df5e81cec: Waiting for close lock at 1732000407523Disabling compacts and flushes for region at 1732000407523Disabling writes for close at 1732000407524 (+1 ms)Writing region close event to WAL at 1732000407524Closed at 1732000407524 2024-11-19T07:13:27,524 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 9357ae6c6e8337f25dacbf0a10701ef6, NAME => 'test,uuu,1732000407304.9357ae6c6e8337f25dacbf0a10701ef6.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:27,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741857_1033 (size=45) 2024-11-19T07:13:27,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741857_1033 (size=45) 2024-11-19T07:13:27,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741857_1033 (size=45) 2024-11-19T07:13:27,532 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ttt,1732000407304.2f38687f659a16d9e2a781f2a75d514a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:27,532 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 2f38687f659a16d9e2a781f2a75d514a, disabling compactions & flushes 2024-11-19T07:13:27,532 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ttt,1732000407304.2f38687f659a16d9e2a781f2a75d514a. 2024-11-19T07:13:27,532 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732000407304.2f38687f659a16d9e2a781f2a75d514a. 2024-11-19T07:13:27,532 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732000407304.2f38687f659a16d9e2a781f2a75d514a. after waiting 0 ms 2024-11-19T07:13:27,532 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732000407304.2f38687f659a16d9e2a781f2a75d514a. 2024-11-19T07:13:27,533 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ttt,1732000407304.2f38687f659a16d9e2a781f2a75d514a. 2024-11-19T07:13:27,533 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2f38687f659a16d9e2a781f2a75d514a: Waiting for close lock at 1732000407532Disabling compacts and flushes for region at 1732000407532Disabling writes for close at 1732000407532Writing region close event to WAL at 1732000407532Closed at 1732000407532 2024-11-19T07:13:27,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741856_1032 (size=45) 2024-11-19T07:13:27,533 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 08f6487d04261d6f0713b4b5adeacf2e, NAME => 'test,vvv,1732000407304.08f6487d04261d6f0713b4b5adeacf2e.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:27,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741856_1032 (size=45) 2024-11-19T07:13:27,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741856_1032 (size=45) 2024-11-19T07:13:27,537 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,sss,1732000407304.d703151d82f3a429f348967988283542.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:27,537 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing d703151d82f3a429f348967988283542, disabling compactions & flushes 2024-11-19T07:13:27,537 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,sss,1732000407304.d703151d82f3a429f348967988283542. 2024-11-19T07:13:27,537 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732000407304.d703151d82f3a429f348967988283542. 2024-11-19T07:13:27,537 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732000407304.d703151d82f3a429f348967988283542. after waiting 0 ms 2024-11-19T07:13:27,537 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732000407304.d703151d82f3a429f348967988283542. 2024-11-19T07:13:27,537 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,sss,1732000407304.d703151d82f3a429f348967988283542. 2024-11-19T07:13:27,537 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for d703151d82f3a429f348967988283542: Waiting for close lock at 1732000407537Disabling compacts and flushes for region at 1732000407537Disabling writes for close at 1732000407537Writing region close event to WAL at 1732000407537Closed at 1732000407537 2024-11-19T07:13:27,538 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 611b07fbb4fb5e3ea7255d9fa0f9fe40, NAME => 'test,www,1732000407304.611b07fbb4fb5e3ea7255d9fa0f9fe40.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:27,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741858_1034 (size=45) 2024-11-19T07:13:27,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741858_1034 (size=45) 2024-11-19T07:13:27,550 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,uuu,1732000407304.9357ae6c6e8337f25dacbf0a10701ef6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:27,550 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 9357ae6c6e8337f25dacbf0a10701ef6, disabling compactions & flushes 2024-11-19T07:13:27,550 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,uuu,1732000407304.9357ae6c6e8337f25dacbf0a10701ef6. 2024-11-19T07:13:27,550 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732000407304.9357ae6c6e8337f25dacbf0a10701ef6. 2024-11-19T07:13:27,550 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732000407304.9357ae6c6e8337f25dacbf0a10701ef6. after waiting 0 ms 2024-11-19T07:13:27,551 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732000407304.9357ae6c6e8337f25dacbf0a10701ef6. 2024-11-19T07:13:27,551 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,uuu,1732000407304.9357ae6c6e8337f25dacbf0a10701ef6. 2024-11-19T07:13:27,551 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 9357ae6c6e8337f25dacbf0a10701ef6: Waiting for close lock at 1732000407550Disabling compacts and flushes for region at 1732000407550Disabling writes for close at 1732000407551 (+1 ms)Writing region close event to WAL at 1732000407551Closed at 1732000407551 2024-11-19T07:13:27,552 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 126b77a274d2ba63b12dd23c9e7ababb, NAME => 'test,xxx,1732000407304.126b77a274d2ba63b12dd23c9e7ababb.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:27,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741858_1034 (size=45) 2024-11-19T07:13:27,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741859_1035 (size=45) 2024-11-19T07:13:27,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741860_1036 (size=45) 2024-11-19T07:13:27,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741859_1035 (size=45) 2024-11-19T07:13:27,575 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,vvv,1732000407304.08f6487d04261d6f0713b4b5adeacf2e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:27,575 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 08f6487d04261d6f0713b4b5adeacf2e, disabling compactions & flushes 2024-11-19T07:13:27,575 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,vvv,1732000407304.08f6487d04261d6f0713b4b5adeacf2e. 2024-11-19T07:13:27,575 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732000407304.08f6487d04261d6f0713b4b5adeacf2e. 2024-11-19T07:13:27,575 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732000407304.08f6487d04261d6f0713b4b5adeacf2e. after waiting 0 ms 2024-11-19T07:13:27,575 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732000407304.08f6487d04261d6f0713b4b5adeacf2e. 2024-11-19T07:13:27,575 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,vvv,1732000407304.08f6487d04261d6f0713b4b5adeacf2e. 2024-11-19T07:13:27,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741859_1035 (size=45) 2024-11-19T07:13:27,576 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 08f6487d04261d6f0713b4b5adeacf2e: Waiting for close lock at 1732000407575Disabling compacts and flushes for region at 1732000407575Disabling writes for close at 1732000407575Writing region close event to WAL at 1732000407575Closed at 1732000407575 2024-11-19T07:13:27,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741860_1036 (size=45) 2024-11-19T07:13:27,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741860_1036 (size=45) 2024-11-19T07:13:27,576 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3aa6780c71c61c778946d2086357143d, NAME => 'test,yyy,1732000407304.3aa6780c71c61c778946d2086357143d.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:27,577 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,www,1732000407304.611b07fbb4fb5e3ea7255d9fa0f9fe40.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:27,577 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 611b07fbb4fb5e3ea7255d9fa0f9fe40, disabling compactions & flushes 2024-11-19T07:13:27,577 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,www,1732000407304.611b07fbb4fb5e3ea7255d9fa0f9fe40. 2024-11-19T07:13:27,577 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732000407304.611b07fbb4fb5e3ea7255d9fa0f9fe40. 2024-11-19T07:13:27,577 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1732000407304.611b07fbb4fb5e3ea7255d9fa0f9fe40. after waiting 0 ms 2024-11-19T07:13:27,578 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1732000407304.611b07fbb4fb5e3ea7255d9fa0f9fe40. 2024-11-19T07:13:27,578 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,www,1732000407304.611b07fbb4fb5e3ea7255d9fa0f9fe40. 2024-11-19T07:13:27,578 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 611b07fbb4fb5e3ea7255d9fa0f9fe40: Waiting for close lock at 1732000407577Disabling compacts and flushes for region at 1732000407577Disabling writes for close at 1732000407577Writing region close event to WAL at 1732000407578 (+1 ms)Closed at 1732000407578 2024-11-19T07:13:27,578 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 32d987cca115e6f476cdb5686be2b726, NAME => 'test,zzz,1732000407304.32d987cca115e6f476cdb5686be2b726.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d 2024-11-19T07:13:27,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741861_1037 (size=45) 2024-11-19T07:13:27,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741861_1037 (size=45) 2024-11-19T07:13:27,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741861_1037 (size=45) 2024-11-19T07:13:27,595 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,xxx,1732000407304.126b77a274d2ba63b12dd23c9e7ababb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:27,595 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 126b77a274d2ba63b12dd23c9e7ababb, disabling compactions & flushes 2024-11-19T07:13:27,595 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,xxx,1732000407304.126b77a274d2ba63b12dd23c9e7ababb. 2024-11-19T07:13:27,595 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732000407304.126b77a274d2ba63b12dd23c9e7ababb. 2024-11-19T07:13:27,595 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732000407304.126b77a274d2ba63b12dd23c9e7ababb. after waiting 0 ms 2024-11-19T07:13:27,595 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732000407304.126b77a274d2ba63b12dd23c9e7ababb. 2024-11-19T07:13:27,595 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,xxx,1732000407304.126b77a274d2ba63b12dd23c9e7ababb. 2024-11-19T07:13:27,595 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 126b77a274d2ba63b12dd23c9e7ababb: Waiting for close lock at 1732000407595Disabling compacts and flushes for region at 1732000407595Disabling writes for close at 1732000407595Writing region close event to WAL at 1732000407595Closed at 1732000407595 2024-11-19T07:13:27,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741863_1039 (size=42) 2024-11-19T07:13:27,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741862_1038 (size=45) 2024-11-19T07:13:27,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741863_1039 (size=42) 2024-11-19T07:13:27,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741862_1038 (size=45) 2024-11-19T07:13:27,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741863_1039 (size=42) 2024-11-19T07:13:27,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741862_1038 (size=45) 2024-11-19T07:13:27,603 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,yyy,1732000407304.3aa6780c71c61c778946d2086357143d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:27,603 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,zzz,1732000407304.32d987cca115e6f476cdb5686be2b726.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:27,603 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 3aa6780c71c61c778946d2086357143d, disabling compactions & flushes 2024-11-19T07:13:27,603 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 32d987cca115e6f476cdb5686be2b726, disabling compactions & flushes 2024-11-19T07:13:27,603 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,yyy,1732000407304.3aa6780c71c61c778946d2086357143d. 2024-11-19T07:13:27,603 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,zzz,1732000407304.32d987cca115e6f476cdb5686be2b726. 2024-11-19T07:13:27,603 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732000407304.3aa6780c71c61c778946d2086357143d. 2024-11-19T07:13:27,603 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732000407304.32d987cca115e6f476cdb5686be2b726. 2024-11-19T07:13:27,603 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732000407304.3aa6780c71c61c778946d2086357143d. after waiting 0 ms 2024-11-19T07:13:27,603 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732000407304.32d987cca115e6f476cdb5686be2b726. after waiting 0 ms 2024-11-19T07:13:27,603 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732000407304.3aa6780c71c61c778946d2086357143d. 2024-11-19T07:13:27,603 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732000407304.32d987cca115e6f476cdb5686be2b726. 2024-11-19T07:13:27,603 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,yyy,1732000407304.3aa6780c71c61c778946d2086357143d. 2024-11-19T07:13:27,603 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,zzz,1732000407304.32d987cca115e6f476cdb5686be2b726. 2024-11-19T07:13:27,603 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3aa6780c71c61c778946d2086357143d: Waiting for close lock at 1732000407603Disabling compacts and flushes for region at 1732000407603Disabling writes for close at 1732000407603Writing region close event to WAL at 1732000407603Closed at 1732000407603 2024-11-19T07:13:27,604 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 32d987cca115e6f476cdb5686be2b726: Waiting for close lock at 1732000407603Disabling compacts and flushes for region at 1732000407603Disabling writes for close at 1732000407603Writing region close event to WAL at 1732000407603Closed at 1732000407603 2024-11-19T07:13:27,607 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T07:13:27,612 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1732000407304.7d9d31e1a2abc752a2056ed0e3a3fcec.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732000407607"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732000407607"}]},"ts":"1732000407607"} 2024-11-19T07:13:27,613 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1732000407304.02309bab61f6207d73b9f19985e1fdb9.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732000407607"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732000407607"}]},"ts":"1732000407607"} 2024-11-19T07:13:27,613 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1732000407304.12dcc969f79695b5f0d224b605268d2d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732000407607"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732000407607"}]},"ts":"1732000407607"} 2024-11-19T07:13:27,613 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1732000407304.e880c03f4c433baa6242dd65ef2d8486.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732000407607"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732000407607"}]},"ts":"1732000407607"} 2024-11-19T07:13:27,613 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1732000407304.1ece759217c055d9a295034ce69dd86d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732000407607"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732000407607"}]},"ts":"1732000407607"} 2024-11-19T07:13:27,613 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1732000407304.49b850b07eedb76caa94e4c93ac5c832.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732000407607"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732000407607"}]},"ts":"1732000407607"} 2024-11-19T07:13:27,614 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1732000407304.66fe58970f9c4a1cba9c3e3da843ed3d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732000407607"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732000407607"}]},"ts":"1732000407607"} 2024-11-19T07:13:27,614 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1732000407304.db4ba820395e8b4a2b38d57f8a222e97.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732000407607"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732000407607"}]},"ts":"1732000407607"} 2024-11-19T07:13:27,614 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1732000407304.43676ec232db06c9bf4af64db1ede4f5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732000407607"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732000407607"}]},"ts":"1732000407607"} 2024-11-19T07:13:27,614 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1732000407304.bbb518f4693490310eca052bf28f87fc.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732000407607"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732000407607"}]},"ts":"1732000407607"} 2024-11-19T07:13:27,615 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1732000407304.4421c1189c5fdff3a40dcc58728c41ab.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732000407607"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732000407607"}]},"ts":"1732000407607"} 2024-11-19T07:13:27,615 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1732000407304.82941673e9ad03e5a037091750c3f315.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732000407607"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732000407607"}]},"ts":"1732000407607"} 2024-11-19T07:13:27,615 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1732000407304.74657be6232ad22aaca548d82491de41.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732000407607"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732000407607"}]},"ts":"1732000407607"} 2024-11-19T07:13:27,615 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1732000407304.d2ef4078aa4f5f4b6f4dc7188dd2f1cb.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732000407607"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732000407607"}]},"ts":"1732000407607"} 2024-11-19T07:13:27,615 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1732000407304.fdf73eccbee4f01f5a86b87f17dcdb8b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732000407607"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732000407607"}]},"ts":"1732000407607"} 2024-11-19T07:13:27,615 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1732000407304.2df93dc30d9c55d765ada1f50e320789.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732000407607"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732000407607"}]},"ts":"1732000407607"} 2024-11-19T07:13:27,615 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1732000407304.3a637105eb1d8b75696b3cfabe5d8173.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732000407607"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732000407607"}]},"ts":"1732000407607"} 2024-11-19T07:13:27,616 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1732000407304.dba97405d3b35647d42d266df5e81cec.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732000407607"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732000407607"}]},"ts":"1732000407607"} 2024-11-19T07:13:27,616 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1732000407304.2f38687f659a16d9e2a781f2a75d514a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732000407607"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732000407607"}]},"ts":"1732000407607"} 2024-11-19T07:13:27,616 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1732000407304.d703151d82f3a429f348967988283542.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732000407607"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732000407607"}]},"ts":"1732000407607"} 2024-11-19T07:13:27,616 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1732000407304.9357ae6c6e8337f25dacbf0a10701ef6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732000407607"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732000407607"}]},"ts":"1732000407607"} 2024-11-19T07:13:27,616 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1732000407304.08f6487d04261d6f0713b4b5adeacf2e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732000407607"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732000407607"}]},"ts":"1732000407607"} 2024-11-19T07:13:27,616 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1732000407304.611b07fbb4fb5e3ea7255d9fa0f9fe40.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732000407607"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732000407607"}]},"ts":"1732000407607"} 2024-11-19T07:13:27,616 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1732000407304.126b77a274d2ba63b12dd23c9e7ababb.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732000407607"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732000407607"}]},"ts":"1732000407607"} 2024-11-19T07:13:27,616 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1732000407304.3aa6780c71c61c778946d2086357143d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732000407607"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732000407607"}]},"ts":"1732000407607"} 2024-11-19T07:13:27,617 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1732000407304.32d987cca115e6f476cdb5686be2b726.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732000407607"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732000407607"}]},"ts":"1732000407607"} 2024-11-19T07:13:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T07:13:27,661 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-19T07:13:27,663 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T07:13:27,666 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732000407663"}]},"ts":"1732000407663"} 2024-11-19T07:13:27,670 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-19T07:13:27,671 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {db0afee3eab9=0} racks are {/default-rack=0} 2024-11-19T07:13:27,688 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-19T07:13:27,689 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-19T07:13:27,689 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-19T07:13:27,689 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-19T07:13:27,689 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-19T07:13:27,689 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-19T07:13:27,689 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-19T07:13:27,689 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-19T07:13:27,689 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-19T07:13:27,689 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-19T07:13:27,692 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=02309bab61f6207d73b9f19985e1fdb9, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=12dcc969f79695b5f0d224b605268d2d, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1ece759217c055d9a295034ce69dd86d, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=49b850b07eedb76caa94e4c93ac5c832, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=db4ba820395e8b4a2b38d57f8a222e97, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=43676ec232db06c9bf4af64db1ede4f5, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4421c1189c5fdff3a40dcc58728c41ab, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=82941673e9ad03e5a037091750c3f315, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fdf73eccbee4f01f5a86b87f17dcdb8b, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3a637105eb1d8b75696b3cfabe5d8173, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dba97405d3b35647d42d266df5e81cec, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d703151d82f3a429f348967988283542, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9357ae6c6e8337f25dacbf0a10701ef6, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=611b07fbb4fb5e3ea7255d9fa0f9fe40, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=126b77a274d2ba63b12dd23c9e7ababb, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=32d987cca115e6f476cdb5686be2b726, ASSIGN}] 2024-11-19T07:13:27,701 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=32d987cca115e6f476cdb5686be2b726, ASSIGN 2024-11-19T07:13:27,702 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=32d987cca115e6f476cdb5686be2b726, ASSIGN; state=OFFLINE, location=db0afee3eab9,42435,1732000404708; forceNewPlan=false, retain=false 2024-11-19T07:13:27,704 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:27,705 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN; state=OFFLINE, location=db0afee3eab9,37853,1732000404756; forceNewPlan=false, retain=false 2024-11-19T07:13:27,707 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=126b77a274d2ba63b12dd23c9e7ababb, ASSIGN 2024-11-19T07:13:27,708 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=126b77a274d2ba63b12dd23c9e7ababb, ASSIGN; state=OFFLINE, location=db0afee3eab9,34425,1732000404604; forceNewPlan=false, retain=false 2024-11-19T07:13:27,710 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=611b07fbb4fb5e3ea7255d9fa0f9fe40, ASSIGN 2024-11-19T07:13:27,711 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=611b07fbb4fb5e3ea7255d9fa0f9fe40, ASSIGN; state=OFFLINE, location=db0afee3eab9,42435,1732000404708; forceNewPlan=false, retain=false 2024-11-19T07:13:27,712 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:27,714 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN; state=OFFLINE, location=db0afee3eab9,37853,1732000404756; forceNewPlan=false, retain=false 2024-11-19T07:13:27,716 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9357ae6c6e8337f25dacbf0a10701ef6, ASSIGN 2024-11-19T07:13:27,717 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9357ae6c6e8337f25dacbf0a10701ef6, ASSIGN; state=OFFLINE, location=db0afee3eab9,34425,1732000404604; forceNewPlan=false, retain=false 2024-11-19T07:13:27,719 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:27,721 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN; state=OFFLINE, location=db0afee3eab9,37853,1732000404756; forceNewPlan=false, retain=false 2024-11-19T07:13:27,723 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d703151d82f3a429f348967988283542, ASSIGN 2024-11-19T07:13:27,724 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d703151d82f3a429f348967988283542, ASSIGN; state=OFFLINE, location=db0afee3eab9,42435,1732000404708; forceNewPlan=false, retain=false 2024-11-19T07:13:27,725 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dba97405d3b35647d42d266df5e81cec, ASSIGN 2024-11-19T07:13:27,727 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=dba97405d3b35647d42d266df5e81cec, ASSIGN; state=OFFLINE, location=db0afee3eab9,34425,1732000404604; forceNewPlan=false, retain=false 2024-11-19T07:13:27,728 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN 2024-11-19T07:13:27,729 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN; state=OFFLINE, location=db0afee3eab9,37853,1732000404756; forceNewPlan=false, retain=false 2024-11-19T07:13:27,731 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3a637105eb1d8b75696b3cfabe5d8173, ASSIGN 2024-11-19T07:13:27,732 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3a637105eb1d8b75696b3cfabe5d8173, ASSIGN; state=OFFLINE, location=db0afee3eab9,42435,1732000404708; forceNewPlan=false, retain=false 2024-11-19T07:13:27,733 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fdf73eccbee4f01f5a86b87f17dcdb8b, ASSIGN 2024-11-19T07:13:27,735 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fdf73eccbee4f01f5a86b87f17dcdb8b, ASSIGN; state=OFFLINE, location=db0afee3eab9,34425,1732000404604; forceNewPlan=false, retain=false 2024-11-19T07:13:27,736 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, ASSIGN 2024-11-19T07:13:27,737 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, ASSIGN; state=OFFLINE, location=db0afee3eab9,37853,1732000404756; forceNewPlan=false, retain=false 2024-11-19T07:13:27,738 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, ASSIGN 2024-11-19T07:13:27,739 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, ASSIGN; state=OFFLINE, location=db0afee3eab9,42435,1732000404708; forceNewPlan=false, retain=false 2024-11-19T07:13:27,741 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=82941673e9ad03e5a037091750c3f315, ASSIGN 2024-11-19T07:13:27,742 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=82941673e9ad03e5a037091750c3f315, ASSIGN; state=OFFLINE, location=db0afee3eab9,34425,1732000404604; forceNewPlan=false, retain=false 2024-11-19T07:13:27,743 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN 2024-11-19T07:13:27,744 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN; state=OFFLINE, location=db0afee3eab9,37853,1732000404756; forceNewPlan=false, retain=false 2024-11-19T07:13:27,746 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4421c1189c5fdff3a40dcc58728c41ab, ASSIGN 2024-11-19T07:13:27,747 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4421c1189c5fdff3a40dcc58728c41ab, ASSIGN; state=OFFLINE, location=db0afee3eab9,42435,1732000404708; forceNewPlan=false, retain=false 2024-11-19T07:13:27,748 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=43676ec232db06c9bf4af64db1ede4f5, ASSIGN 2024-11-19T07:13:27,750 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=43676ec232db06c9bf4af64db1ede4f5, ASSIGN; state=OFFLINE, location=db0afee3eab9,34425,1732000404604; forceNewPlan=false, retain=false 2024-11-19T07:13:27,751 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN 2024-11-19T07:13:27,752 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN; state=OFFLINE, location=db0afee3eab9,37853,1732000404756; forceNewPlan=false, retain=false 2024-11-19T07:13:27,753 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=db4ba820395e8b4a2b38d57f8a222e97, ASSIGN 2024-11-19T07:13:27,754 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=db4ba820395e8b4a2b38d57f8a222e97, ASSIGN; state=OFFLINE, location=db0afee3eab9,42435,1732000404708; forceNewPlan=false, retain=false 2024-11-19T07:13:27,755 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=49b850b07eedb76caa94e4c93ac5c832, ASSIGN 2024-11-19T07:13:27,756 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=49b850b07eedb76caa94e4c93ac5c832, ASSIGN; state=OFFLINE, location=db0afee3eab9,34425,1732000404604; forceNewPlan=false, retain=false 2024-11-19T07:13:27,758 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN 2024-11-19T07:13:27,759 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN; state=OFFLINE, location=db0afee3eab9,37853,1732000404756; forceNewPlan=false, retain=false 2024-11-19T07:13:27,761 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1ece759217c055d9a295034ce69dd86d, ASSIGN 2024-11-19T07:13:27,762 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1ece759217c055d9a295034ce69dd86d, ASSIGN; state=OFFLINE, location=db0afee3eab9,42435,1732000404708; forceNewPlan=false, retain=false 2024-11-19T07:13:27,763 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN 2024-11-19T07:13:27,764 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN; state=OFFLINE, location=db0afee3eab9,37853,1732000404756; forceNewPlan=false, retain=false 2024-11-19T07:13:27,765 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=12dcc969f79695b5f0d224b605268d2d, ASSIGN 2024-11-19T07:13:27,766 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=12dcc969f79695b5f0d224b605268d2d, ASSIGN; state=OFFLINE, location=db0afee3eab9,34425,1732000404604; forceNewPlan=false, retain=false 2024-11-19T07:13:27,767 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=02309bab61f6207d73b9f19985e1fdb9, ASSIGN 2024-11-19T07:13:27,768 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=02309bab61f6207d73b9f19985e1fdb9, ASSIGN; state=OFFLINE, location=db0afee3eab9,42435,1732000404708; forceNewPlan=false, retain=false 2024-11-19T07:13:27,858 INFO [db0afee3eab9:43779 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-19T07:13:27,860 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=32d987cca115e6f476cdb5686be2b726, regionState=OPENING, regionLocation=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:27,861 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=611b07fbb4fb5e3ea7255d9fa0f9fe40, regionState=OPENING, regionLocation=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:27,862 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=d703151d82f3a429f348967988283542, regionState=OPENING, regionLocation=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:27,863 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=3a637105eb1d8b75696b3cfabe5d8173, regionState=OPENING, regionLocation=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:27,864 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, regionState=OPENING, regionLocation=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:27,864 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=32d987cca115e6f476cdb5686be2b726, ASSIGN because future has completed 2024-11-19T07:13:27,864 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=4421c1189c5fdff3a40dcc58728c41ab, regionState=OPENING, regionLocation=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:27,865 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=db4ba820395e8b4a2b38d57f8a222e97, regionState=OPENING, regionLocation=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:27,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32d987cca115e6f476cdb5686be2b726, server=db0afee3eab9,42435,1732000404708}] 2024-11-19T07:13:27,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=611b07fbb4fb5e3ea7255d9fa0f9fe40, ASSIGN because future has completed 2024-11-19T07:13:27,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d703151d82f3a429f348967988283542, ASSIGN because future has completed 2024-11-19T07:13:27,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3a637105eb1d8b75696b3cfabe5d8173, ASSIGN because future has completed 2024-11-19T07:13:27,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a637105eb1d8b75696b3cfabe5d8173, server=db0afee3eab9,42435,1732000404708}] 2024-11-19T07:13:27,873 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, ASSIGN because future has completed 2024-11-19T07:13:27,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4421c1189c5fdff3a40dcc58728c41ab, ASSIGN because future has completed 2024-11-19T07:13:27,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4421c1189c5fdff3a40dcc58728c41ab, server=db0afee3eab9,42435,1732000404708}] 2024-11-19T07:13:27,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=db4ba820395e8b4a2b38d57f8a222e97, ASSIGN because future has completed 2024-11-19T07:13:27,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure db4ba820395e8b4a2b38d57f8a222e97, server=db0afee3eab9,42435,1732000404708}] 2024-11-19T07:13:27,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2ef4078aa4f5f4b6f4dc7188dd2f1cb, server=db0afee3eab9,42435,1732000404708}] 2024-11-19T07:13:27,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure d703151d82f3a429f348967988283542, server=db0afee3eab9,42435,1732000404708}] 2024-11-19T07:13:27,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 611b07fbb4fb5e3ea7255d9fa0f9fe40, server=db0afee3eab9,42435,1732000404708}] 2024-11-19T07:13:27,925 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1ece759217c055d9a295034ce69dd86d, regionState=OPENING, regionLocation=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:27,926 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=02309bab61f6207d73b9f19985e1fdb9, regionState=OPENING, regionLocation=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:27,927 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=3aa6780c71c61c778946d2086357143d, regionState=OPENING, regionLocation=db0afee3eab9,37853,1732000404756 2024-11-19T07:13:27,929 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=08f6487d04261d6f0713b4b5adeacf2e, regionState=OPENING, regionLocation=db0afee3eab9,37853,1732000404756 2024-11-19T07:13:27,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1ece759217c055d9a295034ce69dd86d, ASSIGN because future has completed 2024-11-19T07:13:27,929 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=2f38687f659a16d9e2a781f2a75d514a, regionState=OPENING, regionLocation=db0afee3eab9,37853,1732000404756 2024-11-19T07:13:27,930 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ece759217c055d9a295034ce69dd86d, server=db0afee3eab9,42435,1732000404708}] 2024-11-19T07:13:27,931 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=02309bab61f6207d73b9f19985e1fdb9, ASSIGN because future has completed 2024-11-19T07:13:27,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN because future has completed 2024-11-19T07:13:27,936 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN because future has completed 2024-11-19T07:13:27,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN because future has completed 2024-11-19T07:13:27,939 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,37853,1732000404756}] 2024-11-19T07:13:27,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08f6487d04261d6f0713b4b5adeacf2e, server=db0afee3eab9,37853,1732000404756}] 2024-11-19T07:13:27,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3aa6780c71c61c778946d2086357143d, server=db0afee3eab9,37853,1732000404756}] 2024-11-19T07:13:27,956 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 02309bab61f6207d73b9f19985e1fdb9, server=db0afee3eab9,42435,1732000404708}] 2024-11-19T07:13:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T07:13:27,962 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=2df93dc30d9c55d765ada1f50e320789, regionState=OPENING, regionLocation=db0afee3eab9,37853,1732000404756 2024-11-19T07:13:27,963 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=74657be6232ad22aaca548d82491de41, regionState=OPENING, regionLocation=db0afee3eab9,37853,1732000404756 2024-11-19T07:13:27,964 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=bbb518f4693490310eca052bf28f87fc, regionState=OPENING, regionLocation=db0afee3eab9,37853,1732000404756 2024-11-19T07:13:27,964 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=66fe58970f9c4a1cba9c3e3da843ed3d, regionState=OPENING, regionLocation=db0afee3eab9,37853,1732000404756 2024-11-19T07:13:27,965 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=e880c03f4c433baa6242dd65ef2d8486, regionState=OPENING, regionLocation=db0afee3eab9,37853,1732000404756 2024-11-19T07:13:27,966 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN because future has completed 2024-11-19T07:13:27,966 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7d9d31e1a2abc752a2056ed0e3a3fcec, regionState=OPENING, regionLocation=db0afee3eab9,37853,1732000404756 2024-11-19T07:13:27,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2df93dc30d9c55d765ada1f50e320789, server=db0afee3eab9,37853,1732000404756}] 2024-11-19T07:13:27,968 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, ASSIGN because future has completed 2024-11-19T07:13:27,969 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN because future has completed 2024-11-19T07:13:27,971 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN because future has completed 2024-11-19T07:13:27,973 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66fe58970f9c4a1cba9c3e3da843ed3d, server=db0afee3eab9,37853,1732000404756}] 2024-11-19T07:13:27,973 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN because future has completed 2024-11-19T07:13:27,975 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN because future has completed 2024-11-19T07:13:27,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7d9d31e1a2abc752a2056ed0e3a3fcec, server=db0afee3eab9,37853,1732000404756}] 2024-11-19T07:13:27,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure e880c03f4c433baa6242dd65ef2d8486, server=db0afee3eab9,37853,1732000404756}] 2024-11-19T07:13:27,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbb518f4693490310eca052bf28f87fc, server=db0afee3eab9,37853,1732000404756}] 2024-11-19T07:13:27,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 74657be6232ad22aaca548d82491de41, server=db0afee3eab9,37853,1732000404756}] 2024-11-19T07:13:28,000 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=126b77a274d2ba63b12dd23c9e7ababb, regionState=OPENING, regionLocation=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:28,001 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=9357ae6c6e8337f25dacbf0a10701ef6, regionState=OPENING, regionLocation=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:28,002 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=dba97405d3b35647d42d266df5e81cec, regionState=OPENING, regionLocation=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:28,003 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=fdf73eccbee4f01f5a86b87f17dcdb8b, regionState=OPENING, regionLocation=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:28,003 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=126b77a274d2ba63b12dd23c9e7ababb, ASSIGN because future has completed 2024-11-19T07:13:28,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 126b77a274d2ba63b12dd23c9e7ababb, server=db0afee3eab9,34425,1732000404604}] 2024-11-19T07:13:28,005 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9357ae6c6e8337f25dacbf0a10701ef6, ASSIGN because future has completed 2024-11-19T07:13:28,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=dba97405d3b35647d42d266df5e81cec, ASSIGN because future has completed 2024-11-19T07:13:28,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fdf73eccbee4f01f5a86b87f17dcdb8b, ASSIGN because future has completed 2024-11-19T07:13:28,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure fdf73eccbee4f01f5a86b87f17dcdb8b, server=db0afee3eab9,34425,1732000404604}] 2024-11-19T07:13:28,017 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure dba97405d3b35647d42d266df5e81cec, server=db0afee3eab9,34425,1732000404604}] 2024-11-19T07:13:28,020 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T07:13:28,022 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43685, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T07:13:28,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9357ae6c6e8337f25dacbf0a10701ef6, server=db0afee3eab9,34425,1732000404604}] 2024-11-19T07:13:28,028 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,zzz,1732000407304.32d987cca115e6f476cdb5686be2b726. 2024-11-19T07:13:28,028 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=82941673e9ad03e5a037091750c3f315, regionState=OPENING, regionLocation=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:28,028 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 32d987cca115e6f476cdb5686be2b726, NAME => 'test,zzz,1732000407304.32d987cca115e6f476cdb5686be2b726.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-19T07:13:28,029 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=43676ec232db06c9bf4af64db1ede4f5, regionState=OPENING, regionLocation=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:28,029 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 32d987cca115e6f476cdb5686be2b726 2024-11-19T07:13:28,029 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,zzz,1732000407304.32d987cca115e6f476cdb5686be2b726.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:28,029 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 32d987cca115e6f476cdb5686be2b726 2024-11-19T07:13:28,029 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 32d987cca115e6f476cdb5686be2b726 2024-11-19T07:13:28,030 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=49b850b07eedb76caa94e4c93ac5c832, regionState=OPENING, regionLocation=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:28,031 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=12dcc969f79695b5f0d224b605268d2d, regionState=OPENING, regionLocation=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:28,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=82941673e9ad03e5a037091750c3f315, ASSIGN because future has completed 2024-11-19T07:13:28,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82941673e9ad03e5a037091750c3f315, server=db0afee3eab9,34425,1732000404604}] 2024-11-19T07:13:28,032 INFO [StoreOpener-32d987cca115e6f476cdb5686be2b726-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 32d987cca115e6f476cdb5686be2b726 2024-11-19T07:13:28,033 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=43676ec232db06c9bf4af64db1ede4f5, ASSIGN because future has completed 2024-11-19T07:13:28,034 INFO [StoreOpener-32d987cca115e6f476cdb5686be2b726-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 32d987cca115e6f476cdb5686be2b726 columnFamilyName family 2024-11-19T07:13:28,035 DEBUG [StoreOpener-32d987cca115e6f476cdb5686be2b726-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:28,035 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=49b850b07eedb76caa94e4c93ac5c832, ASSIGN because future has completed 2024-11-19T07:13:28,036 INFO [StoreOpener-32d987cca115e6f476cdb5686be2b726-1 {}] regionserver.HStore(327): Store=32d987cca115e6f476cdb5686be2b726/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:28,036 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 32d987cca115e6f476cdb5686be2b726 2024-11-19T07:13:28,037 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=12dcc969f79695b5f0d224b605268d2d, ASSIGN because future has completed 2024-11-19T07:13:28,037 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12dcc969f79695b5f0d224b605268d2d, server=db0afee3eab9,34425,1732000404604}] 2024-11-19T07:13:28,037 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/32d987cca115e6f476cdb5686be2b726 2024-11-19T07:13:28,038 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/32d987cca115e6f476cdb5686be2b726 2024-11-19T07:13:28,038 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 32d987cca115e6f476cdb5686be2b726 2024-11-19T07:13:28,038 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 32d987cca115e6f476cdb5686be2b726 2024-11-19T07:13:28,041 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 32d987cca115e6f476cdb5686be2b726 2024-11-19T07:13:28,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49b850b07eedb76caa94e4c93ac5c832, server=db0afee3eab9,34425,1732000404604}] 2024-11-19T07:13:28,044 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/32d987cca115e6f476cdb5686be2b726/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T07:13:28,045 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 32d987cca115e6f476cdb5686be2b726; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65783104, jitterRate=-0.01975536346435547}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:28,045 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 32d987cca115e6f476cdb5686be2b726 2024-11-19T07:13:28,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43676ec232db06c9bf4af64db1ede4f5, server=db0afee3eab9,34425,1732000404604}] 2024-11-19T07:13:28,047 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 32d987cca115e6f476cdb5686be2b726: Running coprocessor pre-open hook at 1732000408030Writing region info on filesystem at 1732000408030Initializing all the Stores at 1732000408031 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000408031Cleaning up temporary data from old regions at 1732000408038 (+7 ms)Running coprocessor post-open hooks at 1732000408045 (+7 ms)Region opened successfully at 1732000408047 (+2 ms) 2024-11-19T07:13:28,049 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1732000407304.32d987cca115e6f476cdb5686be2b726., pid=31, masterSystemTime=1732000408019 2024-11-19T07:13:28,052 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1732000407304.32d987cca115e6f476cdb5686be2b726. 2024-11-19T07:13:28,052 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,zzz,1732000407304.32d987cca115e6f476cdb5686be2b726. 2024-11-19T07:13:28,052 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,ppp,1732000407304.3a637105eb1d8b75696b3cfabe5d8173. 2024-11-19T07:13:28,052 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 3a637105eb1d8b75696b3cfabe5d8173, NAME => 'test,ppp,1732000407304.3a637105eb1d8b75696b3cfabe5d8173.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-19T07:13:28,053 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3a637105eb1d8b75696b3cfabe5d8173 2024-11-19T07:13:28,053 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=32d987cca115e6f476cdb5686be2b726, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:28,053 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,ppp,1732000407304.3a637105eb1d8b75696b3cfabe5d8173.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:28,053 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 3a637105eb1d8b75696b3cfabe5d8173 2024-11-19T07:13:28,053 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 3a637105eb1d8b75696b3cfabe5d8173 2024-11-19T07:13:28,056 INFO [StoreOpener-3a637105eb1d8b75696b3cfabe5d8173-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3a637105eb1d8b75696b3cfabe5d8173 2024-11-19T07:13:28,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32d987cca115e6f476cdb5686be2b726, server=db0afee3eab9,42435,1732000404708 because future has completed 2024-11-19T07:13:28,058 INFO [StoreOpener-3a637105eb1d8b75696b3cfabe5d8173-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3a637105eb1d8b75696b3cfabe5d8173 columnFamilyName family 2024-11-19T07:13:28,058 DEBUG [StoreOpener-3a637105eb1d8b75696b3cfabe5d8173-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:28,059 INFO [StoreOpener-3a637105eb1d8b75696b3cfabe5d8173-1 {}] regionserver.HStore(327): Store=3a637105eb1d8b75696b3cfabe5d8173/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:28,059 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 3a637105eb1d8b75696b3cfabe5d8173 2024-11-19T07:13:28,061 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/3a637105eb1d8b75696b3cfabe5d8173 2024-11-19T07:13:28,061 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-19T07:13:28,061 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 32d987cca115e6f476cdb5686be2b726, server=db0afee3eab9,42435,1732000404708 in 192 msec 2024-11-19T07:13:28,062 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/3a637105eb1d8b75696b3cfabe5d8173 2024-11-19T07:13:28,063 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 3a637105eb1d8b75696b3cfabe5d8173 2024-11-19T07:13:28,063 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 3a637105eb1d8b75696b3cfabe5d8173 2024-11-19T07:13:28,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=32d987cca115e6f476cdb5686be2b726, ASSIGN in 370 msec 2024-11-19T07:13:28,065 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 3a637105eb1d8b75696b3cfabe5d8173 2024-11-19T07:13:28,068 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/3a637105eb1d8b75696b3cfabe5d8173/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T07:13:28,069 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 3a637105eb1d8b75696b3cfabe5d8173; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64515448, jitterRate=-0.03864490985870361}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:28,069 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3a637105eb1d8b75696b3cfabe5d8173 2024-11-19T07:13:28,070 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 3a637105eb1d8b75696b3cfabe5d8173: Running coprocessor pre-open hook at 1732000408053Writing region info on filesystem at 1732000408053Initializing all the Stores at 1732000408055 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000408055Cleaning up temporary data from old regions at 1732000408063 (+8 ms)Running coprocessor post-open hooks at 1732000408069 (+6 ms)Region opened successfully at 1732000408070 (+1 ms) 2024-11-19T07:13:28,071 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1732000407304.3a637105eb1d8b75696b3cfabe5d8173., pid=32, masterSystemTime=1732000408019 2024-11-19T07:13:28,073 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1732000407304.3a637105eb1d8b75696b3cfabe5d8173. 2024-11-19T07:13:28,074 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,ppp,1732000407304.3a637105eb1d8b75696b3cfabe5d8173. 2024-11-19T07:13:28,074 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,,1732000407304.02309bab61f6207d73b9f19985e1fdb9. 2024-11-19T07:13:28,074 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 02309bab61f6207d73b9f19985e1fdb9, NAME => 'test,,1732000407304.02309bab61f6207d73b9f19985e1fdb9.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-19T07:13:28,074 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=3a637105eb1d8b75696b3cfabe5d8173, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:28,074 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 02309bab61f6207d73b9f19985e1fdb9 2024-11-19T07:13:28,074 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,,1732000407304.02309bab61f6207d73b9f19985e1fdb9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:28,075 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 02309bab61f6207d73b9f19985e1fdb9 2024-11-19T07:13:28,075 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 02309bab61f6207d73b9f19985e1fdb9 2024-11-19T07:13:28,077 INFO [StoreOpener-02309bab61f6207d73b9f19985e1fdb9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 02309bab61f6207d73b9f19985e1fdb9 2024-11-19T07:13:28,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a637105eb1d8b75696b3cfabe5d8173, server=db0afee3eab9,42435,1732000404708 because future has completed 2024-11-19T07:13:28,079 INFO [StoreOpener-02309bab61f6207d73b9f19985e1fdb9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02309bab61f6207d73b9f19985e1fdb9 columnFamilyName family 2024-11-19T07:13:28,079 DEBUG [StoreOpener-02309bab61f6207d73b9f19985e1fdb9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:28,080 INFO [StoreOpener-02309bab61f6207d73b9f19985e1fdb9-1 {}] regionserver.HStore(327): Store=02309bab61f6207d73b9f19985e1fdb9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:28,081 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 02309bab61f6207d73b9f19985e1fdb9 2024-11-19T07:13:28,082 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/02309bab61f6207d73b9f19985e1fdb9 2024-11-19T07:13:28,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=20 2024-11-19T07:13:28,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 3a637105eb1d8b75696b3cfabe5d8173, server=db0afee3eab9,42435,1732000404708 in 207 msec 2024-11-19T07:13:28,083 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/02309bab61f6207d73b9f19985e1fdb9 2024-11-19T07:13:28,083 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 02309bab61f6207d73b9f19985e1fdb9 2024-11-19T07:13:28,083 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 02309bab61f6207d73b9f19985e1fdb9 2024-11-19T07:13:28,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3a637105eb1d8b75696b3cfabe5d8173, ASSIGN in 392 msec 2024-11-19T07:13:28,086 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 02309bab61f6207d73b9f19985e1fdb9 2024-11-19T07:13:28,089 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/02309bab61f6207d73b9f19985e1fdb9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T07:13:28,089 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 02309bab61f6207d73b9f19985e1fdb9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67157028, jitterRate=7.176995277404785E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:28,090 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 02309bab61f6207d73b9f19985e1fdb9 2024-11-19T07:13:28,090 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 02309bab61f6207d73b9f19985e1fdb9: Running coprocessor pre-open hook at 1732000408075Writing region info on filesystem at 1732000408075Initializing all the Stores at 1732000408076 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000408076Cleaning up temporary data from old regions at 1732000408083 (+7 ms)Running coprocessor post-open hooks at 1732000408090 (+7 ms)Region opened successfully at 1732000408090 2024-11-19T07:13:28,091 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1732000407304.02309bab61f6207d73b9f19985e1fdb9., pid=42, masterSystemTime=1732000408019 2024-11-19T07:13:28,094 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1732000407304.02309bab61f6207d73b9f19985e1fdb9. 2024-11-19T07:13:28,094 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,,1732000407304.02309bab61f6207d73b9f19985e1fdb9. 2024-11-19T07:13:28,094 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,www,1732000407304.611b07fbb4fb5e3ea7255d9fa0f9fe40. 2024-11-19T07:13:28,095 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 611b07fbb4fb5e3ea7255d9fa0f9fe40, NAME => 'test,www,1732000407304.611b07fbb4fb5e3ea7255d9fa0f9fe40.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-19T07:13:28,095 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 611b07fbb4fb5e3ea7255d9fa0f9fe40 2024-11-19T07:13:28,095 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,www,1732000407304.611b07fbb4fb5e3ea7255d9fa0f9fe40.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:28,095 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 611b07fbb4fb5e3ea7255d9fa0f9fe40 2024-11-19T07:13:28,095 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=02309bab61f6207d73b9f19985e1fdb9, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:28,095 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 611b07fbb4fb5e3ea7255d9fa0f9fe40 2024-11-19T07:13:28,098 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,vvv,1732000407304.08f6487d04261d6f0713b4b5adeacf2e. 2024-11-19T07:13:28,098 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 08f6487d04261d6f0713b4b5adeacf2e, NAME => 'test,vvv,1732000407304.08f6487d04261d6f0713b4b5adeacf2e.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-19T07:13:28,099 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 08f6487d04261d6f0713b4b5adeacf2e 2024-11-19T07:13:28,099 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,vvv,1732000407304.08f6487d04261d6f0713b4b5adeacf2e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:28,099 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 08f6487d04261d6f0713b4b5adeacf2e 2024-11-19T07:13:28,099 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 08f6487d04261d6f0713b4b5adeacf2e 2024-11-19T07:13:28,104 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 02309bab61f6207d73b9f19985e1fdb9, server=db0afee3eab9,42435,1732000404708 because future has completed 2024-11-19T07:13:28,105 INFO [StoreOpener-08f6487d04261d6f0713b4b5adeacf2e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 08f6487d04261d6f0713b4b5adeacf2e 2024-11-19T07:13:28,108 INFO [StoreOpener-611b07fbb4fb5e3ea7255d9fa0f9fe40-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 611b07fbb4fb5e3ea7255d9fa0f9fe40 2024-11-19T07:13:28,109 INFO [StoreOpener-08f6487d04261d6f0713b4b5adeacf2e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08f6487d04261d6f0713b4b5adeacf2e columnFamilyName family 2024-11-19T07:13:28,109 DEBUG [StoreOpener-08f6487d04261d6f0713b4b5adeacf2e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:28,110 INFO [StoreOpener-611b07fbb4fb5e3ea7255d9fa0f9fe40-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 611b07fbb4fb5e3ea7255d9fa0f9fe40 columnFamilyName family 2024-11-19T07:13:28,110 DEBUG [StoreOpener-611b07fbb4fb5e3ea7255d9fa0f9fe40-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:28,110 INFO [StoreOpener-08f6487d04261d6f0713b4b5adeacf2e-1 {}] regionserver.HStore(327): Store=08f6487d04261d6f0713b4b5adeacf2e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:28,110 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 08f6487d04261d6f0713b4b5adeacf2e 2024-11-19T07:13:28,111 INFO [StoreOpener-611b07fbb4fb5e3ea7255d9fa0f9fe40-1 {}] regionserver.HStore(327): Store=611b07fbb4fb5e3ea7255d9fa0f9fe40/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:28,111 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 611b07fbb4fb5e3ea7255d9fa0f9fe40 2024-11-19T07:13:28,111 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/08f6487d04261d6f0713b4b5adeacf2e 2024-11-19T07:13:28,112 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/08f6487d04261d6f0713b4b5adeacf2e 2024-11-19T07:13:28,112 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/611b07fbb4fb5e3ea7255d9fa0f9fe40 2024-11-19T07:13:28,112 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/611b07fbb4fb5e3ea7255d9fa0f9fe40 2024-11-19T07:13:28,112 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 08f6487d04261d6f0713b4b5adeacf2e 2024-11-19T07:13:28,112 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 08f6487d04261d6f0713b4b5adeacf2e 2024-11-19T07:13:28,113 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 611b07fbb4fb5e3ea7255d9fa0f9fe40 2024-11-19T07:13:28,113 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 611b07fbb4fb5e3ea7255d9fa0f9fe40 2024-11-19T07:13:28,115 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 08f6487d04261d6f0713b4b5adeacf2e 2024-11-19T07:13:28,115 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 611b07fbb4fb5e3ea7255d9fa0f9fe40 2024-11-19T07:13:28,118 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/611b07fbb4fb5e3ea7255d9fa0f9fe40/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T07:13:28,118 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/08f6487d04261d6f0713b4b5adeacf2e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T07:13:28,119 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 08f6487d04261d6f0713b4b5adeacf2e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75321649, jitterRate=0.12238003313541412}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:28,119 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 611b07fbb4fb5e3ea7255d9fa0f9fe40; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60492485, jitterRate=-0.09859172999858856}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:28,119 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 08f6487d04261d6f0713b4b5adeacf2e 2024-11-19T07:13:28,119 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 611b07fbb4fb5e3ea7255d9fa0f9fe40 2024-11-19T07:13:28,119 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 08f6487d04261d6f0713b4b5adeacf2e: Running coprocessor pre-open hook at 1732000408103Writing region info on filesystem at 1732000408103Initializing all the Stores at 1732000408104 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000408104Cleaning up temporary data from old regions at 1732000408112 (+8 ms)Running coprocessor post-open hooks at 1732000408119 (+7 ms)Region opened successfully at 1732000408119 2024-11-19T07:13:28,119 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 611b07fbb4fb5e3ea7255d9fa0f9fe40: Running coprocessor pre-open hook at 1732000408103Writing region info on filesystem at 1732000408103Initializing all the Stores at 1732000408104 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000408105 (+1 ms)Cleaning up temporary data from old regions at 1732000408113 (+8 ms)Running coprocessor post-open hooks at 1732000408119 (+6 ms)Region opened successfully at 1732000408119 2024-11-19T07:13:28,120 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732000407304.08f6487d04261d6f0713b4b5adeacf2e., pid=40, masterSystemTime=1732000408092 2024-11-19T07:13:28,120 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1732000407304.611b07fbb4fb5e3ea7255d9fa0f9fe40., pid=37, masterSystemTime=1732000408019 2024-11-19T07:13:28,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=5 2024-11-19T07:13:28,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 02309bab61f6207d73b9f19985e1fdb9, server=db0afee3eab9,42435,1732000404708 in 150 msec 2024-11-19T07:13:28,124 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1732000407304.611b07fbb4fb5e3ea7255d9fa0f9fe40. 2024-11-19T07:13:28,124 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,www,1732000407304.611b07fbb4fb5e3ea7255d9fa0f9fe40. 2024-11-19T07:13:28,124 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,ddd,1732000407304.1ece759217c055d9a295034ce69dd86d. 2024-11-19T07:13:28,124 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 1ece759217c055d9a295034ce69dd86d, NAME => 'test,ddd,1732000407304.1ece759217c055d9a295034ce69dd86d.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-19T07:13:28,125 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1ece759217c055d9a295034ce69dd86d 2024-11-19T07:13:28,125 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,ddd,1732000407304.1ece759217c055d9a295034ce69dd86d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:28,125 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 1ece759217c055d9a295034ce69dd86d 2024-11-19T07:13:28,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=02309bab61f6207d73b9f19985e1fdb9, ASSIGN in 431 msec 2024-11-19T07:13:28,125 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 1ece759217c055d9a295034ce69dd86d 2024-11-19T07:13:28,126 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1732000407304.08f6487d04261d6f0713b4b5adeacf2e. 2024-11-19T07:13:28,126 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,vvv,1732000407304.08f6487d04261d6f0713b4b5adeacf2e. 2024-11-19T07:13:28,127 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,eee,1732000407304.e880c03f4c433baa6242dd65ef2d8486. 2024-11-19T07:13:28,127 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => e880c03f4c433baa6242dd65ef2d8486, NAME => 'test,eee,1732000407304.e880c03f4c433baa6242dd65ef2d8486.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-19T07:13:28,127 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=611b07fbb4fb5e3ea7255d9fa0f9fe40, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:28,127 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e880c03f4c433baa6242dd65ef2d8486 2024-11-19T07:13:28,127 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,eee,1732000407304.e880c03f4c433baa6242dd65ef2d8486.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:28,128 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for e880c03f4c433baa6242dd65ef2d8486 2024-11-19T07:13:28,128 INFO [StoreOpener-1ece759217c055d9a295034ce69dd86d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1ece759217c055d9a295034ce69dd86d 2024-11-19T07:13:28,128 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for e880c03f4c433baa6242dd65ef2d8486 2024-11-19T07:13:28,130 INFO [StoreOpener-e880c03f4c433baa6242dd65ef2d8486-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e880c03f4c433baa6242dd65ef2d8486 2024-11-19T07:13:28,130 INFO [StoreOpener-1ece759217c055d9a295034ce69dd86d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ece759217c055d9a295034ce69dd86d columnFamilyName family 2024-11-19T07:13:28,130 DEBUG [StoreOpener-1ece759217c055d9a295034ce69dd86d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:28,131 INFO [StoreOpener-1ece759217c055d9a295034ce69dd86d-1 {}] regionserver.HStore(327): Store=1ece759217c055d9a295034ce69dd86d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:28,132 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 1ece759217c055d9a295034ce69dd86d 2024-11-19T07:13:28,133 INFO [StoreOpener-e880c03f4c433baa6242dd65ef2d8486-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e880c03f4c433baa6242dd65ef2d8486 columnFamilyName family 2024-11-19T07:13:28,133 DEBUG [StoreOpener-e880c03f4c433baa6242dd65ef2d8486-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:28,133 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/1ece759217c055d9a295034ce69dd86d 2024-11-19T07:13:28,134 INFO [StoreOpener-e880c03f4c433baa6242dd65ef2d8486-1 {}] regionserver.HStore(327): Store=e880c03f4c433baa6242dd65ef2d8486/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:28,134 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for e880c03f4c433baa6242dd65ef2d8486 2024-11-19T07:13:28,134 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/1ece759217c055d9a295034ce69dd86d 2024-11-19T07:13:28,135 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 1ece759217c055d9a295034ce69dd86d 2024-11-19T07:13:28,135 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 1ece759217c055d9a295034ce69dd86d 2024-11-19T07:13:28,135 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/e880c03f4c433baa6242dd65ef2d8486 2024-11-19T07:13:28,135 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 611b07fbb4fb5e3ea7255d9fa0f9fe40, server=db0afee3eab9,42435,1732000404708 because future has completed 2024-11-19T07:13:28,136 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/e880c03f4c433baa6242dd65ef2d8486 2024-11-19T07:13:28,136 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=08f6487d04261d6f0713b4b5adeacf2e, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,37853,1732000404756 2024-11-19T07:13:28,138 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for e880c03f4c433baa6242dd65ef2d8486 2024-11-19T07:13:28,138 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for e880c03f4c433baa6242dd65ef2d8486 2024-11-19T07:13:28,139 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 1ece759217c055d9a295034ce69dd86d 2024-11-19T07:13:28,139 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08f6487d04261d6f0713b4b5adeacf2e, server=db0afee3eab9,37853,1732000404756 because future has completed 2024-11-19T07:13:28,142 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for e880c03f4c433baa6242dd65ef2d8486 2024-11-19T07:13:28,143 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/1ece759217c055d9a295034ce69dd86d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T07:13:28,144 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 1ece759217c055d9a295034ce69dd86d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60311828, jitterRate=-0.1012837290763855}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:28,144 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1ece759217c055d9a295034ce69dd86d 2024-11-19T07:13:28,144 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 1ece759217c055d9a295034ce69dd86d: Running coprocessor pre-open hook at 1732000408125Writing region info on filesystem at 1732000408125Initializing all the Stores at 1732000408127 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000408127Cleaning up temporary data from old regions at 1732000408135 (+8 ms)Running coprocessor post-open hooks at 1732000408144 (+9 ms)Region opened successfully at 1732000408144 2024-11-19T07:13:28,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=27 2024-11-19T07:13:28,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 611b07fbb4fb5e3ea7255d9fa0f9fe40, server=db0afee3eab9,42435,1732000404708 in 222 msec 2024-11-19T07:13:28,146 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1732000407304.1ece759217c055d9a295034ce69dd86d., pid=38, masterSystemTime=1732000408019 2024-11-19T07:13:28,146 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/e880c03f4c433baa6242dd65ef2d8486/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T07:13:28,148 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened e880c03f4c433baa6242dd65ef2d8486; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69561734, jitterRate=0.0365506112575531}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:28,148 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e880c03f4c433baa6242dd65ef2d8486 2024-11-19T07:13:28,148 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for e880c03f4c433baa6242dd65ef2d8486: Running coprocessor pre-open hook at 1732000408128Writing region info on filesystem at 1732000408128Initializing all the Stores at 1732000408130 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000408130Cleaning up temporary data from old regions at 1732000408138 (+8 ms)Running coprocessor post-open hooks at 1732000408148 (+10 ms)Region opened successfully at 1732000408148 2024-11-19T07:13:28,149 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1732000407304.1ece759217c055d9a295034ce69dd86d. 2024-11-19T07:13:28,149 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,ddd,1732000407304.1ece759217c055d9a295034ce69dd86d. 2024-11-19T07:13:28,149 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,mmm,1732000407304.d2ef4078aa4f5f4b6f4dc7188dd2f1cb. 2024-11-19T07:13:28,149 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => d2ef4078aa4f5f4b6f4dc7188dd2f1cb, NAME => 'test,mmm,1732000407304.d2ef4078aa4f5f4b6f4dc7188dd2f1cb.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-19T07:13:28,150 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732000407304.e880c03f4c433baa6242dd65ef2d8486., pid=46, masterSystemTime=1732000408092 2024-11-19T07:13:28,150 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=611b07fbb4fb5e3ea7255d9fa0f9fe40, ASSIGN in 454 msec 2024-11-19T07:13:28,150 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d2ef4078aa4f5f4b6f4dc7188dd2f1cb 2024-11-19T07:13:28,150 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,mmm,1732000407304.d2ef4078aa4f5f4b6f4dc7188dd2f1cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:28,150 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for d2ef4078aa4f5f4b6f4dc7188dd2f1cb 2024-11-19T07:13:28,150 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for d2ef4078aa4f5f4b6f4dc7188dd2f1cb 2024-11-19T07:13:28,152 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1ece759217c055d9a295034ce69dd86d, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:28,152 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732000407304.e880c03f4c433baa6242dd65ef2d8486. 2024-11-19T07:13:28,152 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,eee,1732000407304.e880c03f4c433baa6242dd65ef2d8486. 2024-11-19T07:13:28,153 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,ccc,1732000407304.7d9d31e1a2abc752a2056ed0e3a3fcec. 2024-11-19T07:13:28,153 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 7d9d31e1a2abc752a2056ed0e3a3fcec, NAME => 'test,ccc,1732000407304.7d9d31e1a2abc752a2056ed0e3a3fcec.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-19T07:13:28,153 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7d9d31e1a2abc752a2056ed0e3a3fcec 2024-11-19T07:13:28,153 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,ccc,1732000407304.7d9d31e1a2abc752a2056ed0e3a3fcec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:28,153 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 7d9d31e1a2abc752a2056ed0e3a3fcec 2024-11-19T07:13:28,154 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 7d9d31e1a2abc752a2056ed0e3a3fcec 2024-11-19T07:13:28,154 INFO [StoreOpener-d2ef4078aa4f5f4b6f4dc7188dd2f1cb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d2ef4078aa4f5f4b6f4dc7188dd2f1cb 2024-11-19T07:13:28,156 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ece759217c055d9a295034ce69dd86d, server=db0afee3eab9,42435,1732000404708 because future has completed 2024-11-19T07:13:28,156 INFO [StoreOpener-7d9d31e1a2abc752a2056ed0e3a3fcec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7d9d31e1a2abc752a2056ed0e3a3fcec 2024-11-19T07:13:28,157 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=e880c03f4c433baa6242dd65ef2d8486, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,37853,1732000404756 2024-11-19T07:13:28,159 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T07:13:28,159 INFO [StoreOpener-d2ef4078aa4f5f4b6f4dc7188dd2f1cb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2ef4078aa4f5f4b6f4dc7188dd2f1cb columnFamilyName family 2024-11-19T07:13:28,159 DEBUG [StoreOpener-d2ef4078aa4f5f4b6f4dc7188dd2f1cb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:28,160 INFO [StoreOpener-7d9d31e1a2abc752a2056ed0e3a3fcec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7d9d31e1a2abc752a2056ed0e3a3fcec columnFamilyName family 2024-11-19T07:13:28,160 DEBUG [StoreOpener-7d9d31e1a2abc752a2056ed0e3a3fcec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:28,160 INFO [StoreOpener-d2ef4078aa4f5f4b6f4dc7188dd2f1cb-1 {}] regionserver.HStore(327): Store=d2ef4078aa4f5f4b6f4dc7188dd2f1cb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:28,161 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for d2ef4078aa4f5f4b6f4dc7188dd2f1cb 2024-11-19T07:13:28,162 INFO [StoreOpener-7d9d31e1a2abc752a2056ed0e3a3fcec-1 {}] regionserver.HStore(327): Store=7d9d31e1a2abc752a2056ed0e3a3fcec/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:28,162 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure e880c03f4c433baa6242dd65ef2d8486, server=db0afee3eab9,37853,1732000404756 because future has completed 2024-11-19T07:13:28,162 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 7d9d31e1a2abc752a2056ed0e3a3fcec 2024-11-19T07:13:28,162 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/d2ef4078aa4f5f4b6f4dc7188dd2f1cb 2024-11-19T07:13:28,163 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/d2ef4078aa4f5f4b6f4dc7188dd2f1cb 2024-11-19T07:13:28,163 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/7d9d31e1a2abc752a2056ed0e3a3fcec 2024-11-19T07:13:28,163 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for d2ef4078aa4f5f4b6f4dc7188dd2f1cb 2024-11-19T07:13:28,164 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for d2ef4078aa4f5f4b6f4dc7188dd2f1cb 2024-11-19T07:13:28,164 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/7d9d31e1a2abc752a2056ed0e3a3fcec 2024-11-19T07:13:28,165 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 7d9d31e1a2abc752a2056ed0e3a3fcec 2024-11-19T07:13:28,165 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 7d9d31e1a2abc752a2056ed0e3a3fcec 2024-11-19T07:13:28,167 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for d2ef4078aa4f5f4b6f4dc7188dd2f1cb 2024-11-19T07:13:28,168 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 7d9d31e1a2abc752a2056ed0e3a3fcec 2024-11-19T07:13:28,169 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=8 2024-11-19T07:13:28,169 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1ece759217c055d9a295034ce69dd86d, server=db0afee3eab9,42435,1732000404708 in 233 msec 2024-11-19T07:13:28,172 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58351, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T07:13:28,172 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/d2ef4078aa4f5f4b6f4dc7188dd2f1cb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T07:13:28,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1ece759217c055d9a295034ce69dd86d, ASSIGN in 478 msec 2024-11-19T07:13:28,173 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened d2ef4078aa4f5f4b6f4dc7188dd2f1cb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69366439, jitterRate=0.03364048898220062}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:28,173 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d2ef4078aa4f5f4b6f4dc7188dd2f1cb 2024-11-19T07:13:28,173 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for d2ef4078aa4f5f4b6f4dc7188dd2f1cb: Running coprocessor pre-open hook at 1732000408151Writing region info on filesystem at 1732000408151Initializing all the Stores at 1732000408152 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000408153 (+1 ms)Cleaning up temporary data from old regions at 1732000408164 (+11 ms)Running coprocessor post-open hooks at 1732000408173 (+9 ms)Region opened successfully at 1732000408173 2024-11-19T07:13:28,175 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/7d9d31e1a2abc752a2056ed0e3a3fcec/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T07:13:28,175 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 7d9d31e1a2abc752a2056ed0e3a3fcec; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66383690, jitterRate=-0.01080593466758728}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:28,176 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7d9d31e1a2abc752a2056ed0e3a3fcec 2024-11-19T07:13:28,176 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 7d9d31e1a2abc752a2056ed0e3a3fcec: Running coprocessor pre-open hook at 1732000408154Writing region info on filesystem at 1732000408154Initializing all the Stores at 1732000408155 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000408155Cleaning up temporary data from old regions at 1732000408165 (+10 ms)Running coprocessor post-open hooks at 1732000408176 (+11 ms)Region opened successfully at 1732000408176 2024-11-19T07:13:28,177 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1732000407304.d2ef4078aa4f5f4b6f4dc7188dd2f1cb., pid=35, masterSystemTime=1732000408019 2024-11-19T07:13:28,179 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732000407304.7d9d31e1a2abc752a2056ed0e3a3fcec., pid=45, masterSystemTime=1732000408092 2024-11-19T07:13:28,179 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,xxx,1732000407304.126b77a274d2ba63b12dd23c9e7ababb. 2024-11-19T07:13:28,179 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 126b77a274d2ba63b12dd23c9e7ababb, NAME => 'test,xxx,1732000407304.126b77a274d2ba63b12dd23c9e7ababb.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-19T07:13:28,180 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 126b77a274d2ba63b12dd23c9e7ababb 2024-11-19T07:13:28,180 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,xxx,1732000407304.126b77a274d2ba63b12dd23c9e7ababb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:28,180 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 126b77a274d2ba63b12dd23c9e7ababb 2024-11-19T07:13:28,180 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 126b77a274d2ba63b12dd23c9e7ababb 2024-11-19T07:13:28,181 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1732000407304.d2ef4078aa4f5f4b6f4dc7188dd2f1cb. 2024-11-19T07:13:28,182 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,mmm,1732000407304.d2ef4078aa4f5f4b6f4dc7188dd2f1cb. 2024-11-19T07:13:28,182 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,sss,1732000407304.d703151d82f3a429f348967988283542. 2024-11-19T07:13:28,182 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => d703151d82f3a429f348967988283542, NAME => 'test,sss,1732000407304.d703151d82f3a429f348967988283542.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-19T07:13:28,182 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d703151d82f3a429f348967988283542 2024-11-19T07:13:28,183 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,sss,1732000407304.d703151d82f3a429f348967988283542.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:28,183 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for d703151d82f3a429f348967988283542 2024-11-19T07:13:28,183 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for d703151d82f3a429f348967988283542 2024-11-19T07:13:28,183 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=9 2024-11-19T07:13:28,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure e880c03f4c433baa6242dd65ef2d8486, server=db0afee3eab9,37853,1732000404756 in 191 msec 2024-11-19T07:13:28,185 INFO [StoreOpener-d703151d82f3a429f348967988283542-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d703151d82f3a429f348967988283542 2024-11-19T07:13:28,185 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1732000407304.7d9d31e1a2abc752a2056ed0e3a3fcec. 2024-11-19T07:13:28,185 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,ccc,1732000407304.7d9d31e1a2abc752a2056ed0e3a3fcec. 2024-11-19T07:13:28,185 INFO [StoreOpener-126b77a274d2ba63b12dd23c9e7ababb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 126b77a274d2ba63b12dd23c9e7ababb 2024-11-19T07:13:28,186 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,ttt,1732000407304.2f38687f659a16d9e2a781f2a75d514a. 2024-11-19T07:13:28,186 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 2f38687f659a16d9e2a781f2a75d514a, NAME => 'test,ttt,1732000407304.2f38687f659a16d9e2a781f2a75d514a.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-19T07:13:28,186 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2f38687f659a16d9e2a781f2a75d514a 2024-11-19T07:13:28,186 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,ttt,1732000407304.2f38687f659a16d9e2a781f2a75d514a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:28,186 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 2f38687f659a16d9e2a781f2a75d514a 2024-11-19T07:13:28,187 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 2f38687f659a16d9e2a781f2a75d514a 2024-11-19T07:13:28,187 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN in 493 msec 2024-11-19T07:13:28,188 INFO [StoreOpener-126b77a274d2ba63b12dd23c9e7ababb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 126b77a274d2ba63b12dd23c9e7ababb columnFamilyName family 2024-11-19T07:13:28,188 INFO [StoreOpener-d703151d82f3a429f348967988283542-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d703151d82f3a429f348967988283542 columnFamilyName family 2024-11-19T07:13:28,188 DEBUG [StoreOpener-126b77a274d2ba63b12dd23c9e7ababb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:28,188 DEBUG [StoreOpener-d703151d82f3a429f348967988283542-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:28,189 INFO [StoreOpener-d703151d82f3a429f348967988283542-1 {}] regionserver.HStore(327): Store=d703151d82f3a429f348967988283542/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:28,189 INFO [StoreOpener-126b77a274d2ba63b12dd23c9e7ababb-1 {}] regionserver.HStore(327): Store=126b77a274d2ba63b12dd23c9e7ababb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:28,189 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 126b77a274d2ba63b12dd23c9e7ababb 2024-11-19T07:13:28,189 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for d703151d82f3a429f348967988283542 2024-11-19T07:13:28,189 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7d9d31e1a2abc752a2056ed0e3a3fcec, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,37853,1732000404756 2024-11-19T07:13:28,190 INFO [StoreOpener-2f38687f659a16d9e2a781f2a75d514a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2f38687f659a16d9e2a781f2a75d514a 2024-11-19T07:13:28,190 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/126b77a274d2ba63b12dd23c9e7ababb 2024-11-19T07:13:28,191 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/126b77a274d2ba63b12dd23c9e7ababb 2024-11-19T07:13:28,191 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/d703151d82f3a429f348967988283542 2024-11-19T07:13:28,192 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 126b77a274d2ba63b12dd23c9e7ababb 2024-11-19T07:13:28,192 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 126b77a274d2ba63b12dd23c9e7ababb 2024-11-19T07:13:28,192 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/d703151d82f3a429f348967988283542 2024-11-19T07:13:28,193 INFO [StoreOpener-2f38687f659a16d9e2a781f2a75d514a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2f38687f659a16d9e2a781f2a75d514a columnFamilyName family 2024-11-19T07:13:28,194 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for d703151d82f3a429f348967988283542 2024-11-19T07:13:28,194 DEBUG [StoreOpener-2f38687f659a16d9e2a781f2a75d514a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:28,194 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for d703151d82f3a429f348967988283542 2024-11-19T07:13:28,194 INFO [StoreOpener-2f38687f659a16d9e2a781f2a75d514a-1 {}] regionserver.HStore(327): Store=2f38687f659a16d9e2a781f2a75d514a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:28,195 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 2f38687f659a16d9e2a781f2a75d514a 2024-11-19T07:13:28,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7d9d31e1a2abc752a2056ed0e3a3fcec, server=db0afee3eab9,37853,1732000404756 because future has completed 2024-11-19T07:13:28,195 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:28,196 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 126b77a274d2ba63b12dd23c9e7ababb 2024-11-19T07:13:28,196 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/2f38687f659a16d9e2a781f2a75d514a 2024-11-19T07:13:28,197 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/2f38687f659a16d9e2a781f2a75d514a 2024-11-19T07:13:28,198 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 2f38687f659a16d9e2a781f2a75d514a 2024-11-19T07:13:28,198 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 2f38687f659a16d9e2a781f2a75d514a 2024-11-19T07:13:28,199 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for d703151d82f3a429f348967988283542 2024-11-19T07:13:28,200 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2ef4078aa4f5f4b6f4dc7188dd2f1cb, server=db0afee3eab9,42435,1732000404708 because future has completed 2024-11-19T07:13:28,203 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/d703151d82f3a429f348967988283542/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T07:13:28,203 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 2f38687f659a16d9e2a781f2a75d514a 2024-11-19T07:13:28,203 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened d703151d82f3a429f348967988283542; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68685440, jitterRate=0.023492813110351562}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:28,204 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d703151d82f3a429f348967988283542 2024-11-19T07:13:28,204 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for d703151d82f3a429f348967988283542: Running coprocessor pre-open hook at 1732000408183Writing region info on filesystem at 1732000408183Initializing all the Stores at 1732000408184 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000408184Cleaning up temporary data from old regions at 1732000408194 (+10 ms)Running coprocessor post-open hooks at 1732000408204 (+10 ms)Region opened successfully at 1732000408204 2024-11-19T07:13:28,205 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1732000407304.d703151d82f3a429f348967988283542., pid=36, masterSystemTime=1732000408019 2024-11-19T07:13:28,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=7 2024-11-19T07:13:28,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 7d9d31e1a2abc752a2056ed0e3a3fcec, server=db0afee3eab9,37853,1732000404756 in 221 msec 2024-11-19T07:13:28,207 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/126b77a274d2ba63b12dd23c9e7ababb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T07:13:28,208 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1732000407304.d703151d82f3a429f348967988283542. 2024-11-19T07:13:28,208 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,sss,1732000407304.d703151d82f3a429f348967988283542. 2024-11-19T07:13:28,208 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 126b77a274d2ba63b12dd23c9e7ababb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73114501, jitterRate=0.08949096500873566}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:28,208 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,jjj,1732000407304.4421c1189c5fdff3a40dcc58728c41ab. 2024-11-19T07:13:28,209 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 126b77a274d2ba63b12dd23c9e7ababb 2024-11-19T07:13:28,209 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=db0afee3eab9,37853,1732000404756, table=test, region=08f6487d04261d6f0713b4b5adeacf2e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T07:13:28,209 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 4421c1189c5fdff3a40dcc58728c41ab, NAME => 'test,jjj,1732000407304.4421c1189c5fdff3a40dcc58728c41ab.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-19T07:13:28,209 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 126b77a274d2ba63b12dd23c9e7ababb: Running coprocessor pre-open hook at 1732000408182Writing region info on filesystem at 1732000408182Initializing all the Stores at 1732000408184 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000408184Cleaning up temporary data from old regions at 1732000408192 (+8 ms)Running coprocessor post-open hooks at 1732000408209 (+17 ms)Region opened successfully at 1732000408209 2024-11-19T07:13:28,209 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4421c1189c5fdff3a40dcc58728c41ab 2024-11-19T07:13:28,210 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,jjj,1732000407304.4421c1189c5fdff3a40dcc58728c41ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:28,210 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 4421c1189c5fdff3a40dcc58728c41ab 2024-11-19T07:13:28,210 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 4421c1189c5fdff3a40dcc58728c41ab 2024-11-19T07:13:28,210 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732000407304.126b77a274d2ba63b12dd23c9e7ababb., pid=49, masterSystemTime=1732000408159 2024-11-19T07:13:28,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN in 515 msec 2024-11-19T07:13:28,213 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=d703151d82f3a429f348967988283542, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:28,214 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732000407304.126b77a274d2ba63b12dd23c9e7ababb. 2024-11-19T07:13:28,214 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,xxx,1732000407304.126b77a274d2ba63b12dd23c9e7ababb. 2024-11-19T07:13:28,214 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,bbb,1732000407304.12dcc969f79695b5f0d224b605268d2d. 2024-11-19T07:13:28,215 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 12dcc969f79695b5f0d224b605268d2d, NAME => 'test,bbb,1732000407304.12dcc969f79695b5f0d224b605268d2d.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-19T07:13:28,216 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 12dcc969f79695b5f0d224b605268d2d 2024-11-19T07:13:28,216 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,bbb,1732000407304.12dcc969f79695b5f0d224b605268d2d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:28,216 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 12dcc969f79695b5f0d224b605268d2d 2024-11-19T07:13:28,216 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 12dcc969f79695b5f0d224b605268d2d 2024-11-19T07:13:28,217 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure d703151d82f3a429f348967988283542, server=db0afee3eab9,42435,1732000404708 because future has completed 2024-11-19T07:13:28,218 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=126b77a274d2ba63b12dd23c9e7ababb, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:28,221 INFO [StoreOpener-4421c1189c5fdff3a40dcc58728c41ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4421c1189c5fdff3a40dcc58728c41ab 2024-11-19T07:13:28,221 INFO [StoreOpener-12dcc969f79695b5f0d224b605268d2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 12dcc969f79695b5f0d224b605268d2d 2024-11-19T07:13:28,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 126b77a274d2ba63b12dd23c9e7ababb, server=db0afee3eab9,34425,1732000404604 because future has completed 2024-11-19T07:13:28,224 INFO [StoreOpener-12dcc969f79695b5f0d224b605268d2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12dcc969f79695b5f0d224b605268d2d columnFamilyName family 2024-11-19T07:13:28,224 DEBUG [StoreOpener-12dcc969f79695b5f0d224b605268d2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:28,225 INFO [StoreOpener-4421c1189c5fdff3a40dcc58728c41ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4421c1189c5fdff3a40dcc58728c41ab columnFamilyName family 2024-11-19T07:13:28,225 INFO [StoreOpener-12dcc969f79695b5f0d224b605268d2d-1 {}] regionserver.HStore(327): Store=12dcc969f79695b5f0d224b605268d2d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:28,225 DEBUG [StoreOpener-4421c1189c5fdff3a40dcc58728c41ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:28,226 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 12dcc969f79695b5f0d224b605268d2d 2024-11-19T07:13:28,226 INFO [StoreOpener-4421c1189c5fdff3a40dcc58728c41ab-1 {}] regionserver.HStore(327): Store=4421c1189c5fdff3a40dcc58728c41ab/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:28,227 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/12dcc969f79695b5f0d224b605268d2d 2024-11-19T07:13:28,227 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 4421c1189c5fdff3a40dcc58728c41ab 2024-11-19T07:13:28,227 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/12dcc969f79695b5f0d224b605268d2d 2024-11-19T07:13:28,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=23 2024-11-19T07:13:28,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure d703151d82f3a429f348967988283542, server=db0afee3eab9,42435,1732000404708 in 316 msec 2024-11-19T07:13:28,228 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 12dcc969f79695b5f0d224b605268d2d 2024-11-19T07:13:28,228 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 12dcc969f79695b5f0d224b605268d2d 2024-11-19T07:13:28,230 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/4421c1189c5fdff3a40dcc58728c41ab 2024-11-19T07:13:28,230 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/4421c1189c5fdff3a40dcc58728c41ab 2024-11-19T07:13:28,231 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 4421c1189c5fdff3a40dcc58728c41ab 2024-11-19T07:13:28,231 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 4421c1189c5fdff3a40dcc58728c41ab 2024-11-19T07:13:28,232 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 12dcc969f79695b5f0d224b605268d2d 2024-11-19T07:13:28,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d703151d82f3a429f348967988283542, ASSIGN in 537 msec 2024-11-19T07:13:28,235 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/2f38687f659a16d9e2a781f2a75d514a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T07:13:28,235 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 4421c1189c5fdff3a40dcc58728c41ab 2024-11-19T07:13:28,235 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/12dcc969f79695b5f0d224b605268d2d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T07:13:28,235 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 2f38687f659a16d9e2a781f2a75d514a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63368827, jitterRate=-0.05573089420795441}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:28,236 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2f38687f659a16d9e2a781f2a75d514a 2024-11-19T07:13:28,236 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 2f38687f659a16d9e2a781f2a75d514a: Running coprocessor pre-open hook at 1732000408187Writing region info on filesystem at 1732000408187Initializing all the Stores at 1732000408189 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000408189Cleaning up temporary data from old regions at 1732000408198 (+9 ms)Running coprocessor post-open hooks at 1732000408236 (+38 ms)Region opened successfully at 1732000408236 2024-11-19T07:13:28,237 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1732000407304.2f38687f659a16d9e2a781f2a75d514a., pid=39, masterSystemTime=1732000408092 2024-11-19T07:13:28,239 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 12dcc969f79695b5f0d224b605268d2d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64398665, jitterRate=-0.040385112166404724}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:28,239 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 12dcc969f79695b5f0d224b605268d2d 2024-11-19T07:13:28,239 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 12dcc969f79695b5f0d224b605268d2d: Running coprocessor pre-open hook at 1732000408217Writing region info on filesystem at 1732000408217Initializing all the Stores at 1732000408218 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000408219 (+1 ms)Cleaning up temporary data from old regions at 1732000408228 (+9 ms)Running coprocessor post-open hooks at 1732000408239 (+11 ms)Region opened successfully at 1732000408239 2024-11-19T07:13:28,241 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/4421c1189c5fdff3a40dcc58728c41ab/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T07:13:28,241 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1732000407304.2f38687f659a16d9e2a781f2a75d514a. 2024-11-19T07:13:28,241 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,ttt,1732000407304.2f38687f659a16d9e2a781f2a75d514a. 2024-11-19T07:13:28,242 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,hhh,1732000407304.66fe58970f9c4a1cba9c3e3da843ed3d. 2024-11-19T07:13:28,242 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 66fe58970f9c4a1cba9c3e3da843ed3d, NAME => 'test,hhh,1732000407304.66fe58970f9c4a1cba9c3e3da843ed3d.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-19T07:13:28,242 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 66fe58970f9c4a1cba9c3e3da843ed3d 2024-11-19T07:13:28,242 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 4421c1189c5fdff3a40dcc58728c41ab; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71353811, jitterRate=0.06325463950634003}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:28,242 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,hhh,1732000407304.66fe58970f9c4a1cba9c3e3da843ed3d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:28,242 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4421c1189c5fdff3a40dcc58728c41ab 2024-11-19T07:13:28,243 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 66fe58970f9c4a1cba9c3e3da843ed3d 2024-11-19T07:13:28,243 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 66fe58970f9c4a1cba9c3e3da843ed3d 2024-11-19T07:13:28,243 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 4421c1189c5fdff3a40dcc58728c41ab: Running coprocessor pre-open hook at 1732000408210Writing region info on filesystem at 1732000408210Initializing all the Stores at 1732000408212 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000408212Cleaning up temporary data from old regions at 1732000408231 (+19 ms)Running coprocessor post-open hooks at 1732000408243 (+12 ms)Region opened successfully at 1732000408243 2024-11-19T07:13:28,244 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1732000407304.12dcc969f79695b5f0d224b605268d2d., pid=54, masterSystemTime=1732000408159 2024-11-19T07:13:28,244 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1732000407304.4421c1189c5fdff3a40dcc58728c41ab., pid=33, masterSystemTime=1732000408019 2024-11-19T07:13:28,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=28 2024-11-19T07:13:28,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 126b77a274d2ba63b12dd23c9e7ababb, server=db0afee3eab9,34425,1732000404604 in 231 msec 2024-11-19T07:13:28,246 INFO [StoreOpener-66fe58970f9c4a1cba9c3e3da843ed3d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 66fe58970f9c4a1cba9c3e3da843ed3d 2024-11-19T07:13:28,249 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1732000407304.12dcc969f79695b5f0d224b605268d2d. 2024-11-19T07:13:28,249 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,bbb,1732000407304.12dcc969f79695b5f0d224b605268d2d. 2024-11-19T07:13:28,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=126b77a274d2ba63b12dd23c9e7ababb, ASSIGN in 554 msec 2024-11-19T07:13:28,249 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1732000407304.4421c1189c5fdff3a40dcc58728c41ab. 2024-11-19T07:13:28,249 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,jjj,1732000407304.4421c1189c5fdff3a40dcc58728c41ab. 2024-11-19T07:13:28,250 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,ggg,1732000407304.db4ba820395e8b4a2b38d57f8a222e97. 2024-11-19T07:13:28,250 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => db4ba820395e8b4a2b38d57f8a222e97, NAME => 'test,ggg,1732000407304.db4ba820395e8b4a2b38d57f8a222e97.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-19T07:13:28,250 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test db4ba820395e8b4a2b38d57f8a222e97 2024-11-19T07:13:28,250 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,ggg,1732000407304.db4ba820395e8b4a2b38d57f8a222e97.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:28,250 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for db4ba820395e8b4a2b38d57f8a222e97 2024-11-19T07:13:28,251 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for db4ba820395e8b4a2b38d57f8a222e97 2024-11-19T07:13:28,251 INFO [StoreOpener-66fe58970f9c4a1cba9c3e3da843ed3d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66fe58970f9c4a1cba9c3e3da843ed3d columnFamilyName family 2024-11-19T07:13:28,251 DEBUG [StoreOpener-66fe58970f9c4a1cba9c3e3da843ed3d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:28,251 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,lll,1732000407304.82941673e9ad03e5a037091750c3f315. 2024-11-19T07:13:28,251 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 82941673e9ad03e5a037091750c3f315, NAME => 'test,lll,1732000407304.82941673e9ad03e5a037091750c3f315.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-19T07:13:28,252 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 82941673e9ad03e5a037091750c3f315 2024-11-19T07:13:28,252 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,lll,1732000407304.82941673e9ad03e5a037091750c3f315.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:28,252 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 82941673e9ad03e5a037091750c3f315 2024-11-19T07:13:28,252 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 82941673e9ad03e5a037091750c3f315 2024-11-19T07:13:28,253 INFO [StoreOpener-66fe58970f9c4a1cba9c3e3da843ed3d-1 {}] regionserver.HStore(327): Store=66fe58970f9c4a1cba9c3e3da843ed3d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:28,253 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 66fe58970f9c4a1cba9c3e3da843ed3d 2024-11-19T07:13:28,253 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=4421c1189c5fdff3a40dcc58728c41ab, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:28,254 INFO [StoreOpener-db4ba820395e8b4a2b38d57f8a222e97-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region db4ba820395e8b4a2b38d57f8a222e97 2024-11-19T07:13:28,254 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/66fe58970f9c4a1cba9c3e3da843ed3d 2024-11-19T07:13:28,255 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/66fe58970f9c4a1cba9c3e3da843ed3d 2024-11-19T07:13:28,256 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 66fe58970f9c4a1cba9c3e3da843ed3d 2024-11-19T07:13:28,256 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 66fe58970f9c4a1cba9c3e3da843ed3d 2024-11-19T07:13:28,257 INFO [StoreOpener-db4ba820395e8b4a2b38d57f8a222e97-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db4ba820395e8b4a2b38d57f8a222e97 columnFamilyName family 2024-11-19T07:13:28,258 DEBUG [StoreOpener-db4ba820395e8b4a2b38d57f8a222e97-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:28,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4421c1189c5fdff3a40dcc58728c41ab, server=db0afee3eab9,42435,1732000404708 because future has completed 2024-11-19T07:13:28,259 INFO [StoreOpener-db4ba820395e8b4a2b38d57f8a222e97-1 {}] regionserver.HStore(327): Store=db4ba820395e8b4a2b38d57f8a222e97/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:28,260 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for db4ba820395e8b4a2b38d57f8a222e97 2024-11-19T07:13:28,260 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 66fe58970f9c4a1cba9c3e3da843ed3d 2024-11-19T07:13:28,260 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=12dcc969f79695b5f0d224b605268d2d, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:28,260 INFO [StoreOpener-82941673e9ad03e5a037091750c3f315-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 82941673e9ad03e5a037091750c3f315 2024-11-19T07:13:28,261 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/db4ba820395e8b4a2b38d57f8a222e97 2024-11-19T07:13:28,261 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/db4ba820395e8b4a2b38d57f8a222e97 2024-11-19T07:13:28,263 INFO [StoreOpener-82941673e9ad03e5a037091750c3f315-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82941673e9ad03e5a037091750c3f315 columnFamilyName family 2024-11-19T07:13:28,263 DEBUG [StoreOpener-82941673e9ad03e5a037091750c3f315-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:28,263 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for db4ba820395e8b4a2b38d57f8a222e97 2024-11-19T07:13:28,263 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for db4ba820395e8b4a2b38d57f8a222e97 2024-11-19T07:13:28,264 INFO [StoreOpener-82941673e9ad03e5a037091750c3f315-1 {}] regionserver.HStore(327): Store=82941673e9ad03e5a037091750c3f315/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:28,264 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 82941673e9ad03e5a037091750c3f315 2024-11-19T07:13:28,265 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/66fe58970f9c4a1cba9c3e3da843ed3d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T07:13:28,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12dcc969f79695b5f0d224b605268d2d, server=db0afee3eab9,34425,1732000404604 because future has completed 2024-11-19T07:13:28,265 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/82941673e9ad03e5a037091750c3f315 2024-11-19T07:13:28,266 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/82941673e9ad03e5a037091750c3f315 2024-11-19T07:13:28,266 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 66fe58970f9c4a1cba9c3e3da843ed3d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61784855, jitterRate=-0.07933391630649567}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:28,266 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 66fe58970f9c4a1cba9c3e3da843ed3d 2024-11-19T07:13:28,266 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 66fe58970f9c4a1cba9c3e3da843ed3d: Running coprocessor pre-open hook at 1732000408243Writing region info on filesystem at 1732000408243Initializing all the Stores at 1732000408245 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000408245Cleaning up temporary data from old regions at 1732000408257 (+12 ms)Running coprocessor post-open hooks at 1732000408266 (+9 ms)Region opened successfully at 1732000408266 2024-11-19T07:13:28,267 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 82941673e9ad03e5a037091750c3f315 2024-11-19T07:13:28,267 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 82941673e9ad03e5a037091750c3f315 2024-11-19T07:13:28,267 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for db4ba820395e8b4a2b38d57f8a222e97 2024-11-19T07:13:28,269 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1732000407304.66fe58970f9c4a1cba9c3e3da843ed3d., pid=44, masterSystemTime=1732000408092 2024-11-19T07:13:28,269 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 82941673e9ad03e5a037091750c3f315 2024-11-19T07:13:28,272 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/db4ba820395e8b4a2b38d57f8a222e97/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T07:13:28,274 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=db0afee3eab9,34425,1732000404604, table=test, region=12dcc969f79695b5f0d224b605268d2d. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T07:13:28,274 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened db4ba820395e8b4a2b38d57f8a222e97; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65249512, jitterRate=-0.027706503868103027}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:28,274 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for db4ba820395e8b4a2b38d57f8a222e97 2024-11-19T07:13:28,274 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for db4ba820395e8b4a2b38d57f8a222e97: Running coprocessor pre-open hook at 1732000408251Writing region info on filesystem at 1732000408251Initializing all the Stores at 1732000408253 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000408253Cleaning up temporary data from old regions at 1732000408263 (+10 ms)Running coprocessor post-open hooks at 1732000408274 (+11 ms)Region opened successfully at 1732000408274 2024-11-19T07:13:28,274 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1732000407304.66fe58970f9c4a1cba9c3e3da843ed3d. 2024-11-19T07:13:28,274 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,hhh,1732000407304.66fe58970f9c4a1cba9c3e3da843ed3d. 2024-11-19T07:13:28,275 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,kkk,1732000407304.bbb518f4693490310eca052bf28f87fc. 2024-11-19T07:13:28,275 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => bbb518f4693490310eca052bf28f87fc, NAME => 'test,kkk,1732000407304.bbb518f4693490310eca052bf28f87fc.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-19T07:13:28,275 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bbb518f4693490310eca052bf28f87fc 2024-11-19T07:13:28,276 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,kkk,1732000407304.bbb518f4693490310eca052bf28f87fc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:28,276 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for bbb518f4693490310eca052bf28f87fc 2024-11-19T07:13:28,276 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=db0afee3eab9,42435,1732000404708, table=test, region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T07:13:28,276 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for bbb518f4693490310eca052bf28f87fc 2024-11-19T07:13:28,277 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1732000407304.db4ba820395e8b4a2b38d57f8a222e97., pid=34, masterSystemTime=1732000408019 2024-11-19T07:13:28,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=14 2024-11-19T07:13:28,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 4421c1189c5fdff3a40dcc58728c41ab, server=db0afee3eab9,42435,1732000404708 in 381 msec 2024-11-19T07:13:28,279 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/82941673e9ad03e5a037091750c3f315/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T07:13:28,280 INFO [StoreOpener-bbb518f4693490310eca052bf28f87fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bbb518f4693490310eca052bf28f87fc 2024-11-19T07:13:28,281 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1732000407304.db4ba820395e8b4a2b38d57f8a222e97. 2024-11-19T07:13:28,281 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,ggg,1732000407304.db4ba820395e8b4a2b38d57f8a222e97. 2024-11-19T07:13:28,281 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 82941673e9ad03e5a037091750c3f315; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63976921, jitterRate=-0.04666958749294281}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:28,281 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 82941673e9ad03e5a037091750c3f315 2024-11-19T07:13:28,281 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 82941673e9ad03e5a037091750c3f315: Running coprocessor pre-open hook at 1732000408252Writing region info on filesystem at 1732000408252Initializing all the Stores at 1732000408254 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000408254Cleaning up temporary data from old regions at 1732000408267 (+13 ms)Running coprocessor post-open hooks at 1732000408281 (+14 ms)Region opened successfully at 1732000408281 2024-11-19T07:13:28,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4421c1189c5fdff3a40dcc58728c41ab, ASSIGN in 588 msec 2024-11-19T07:13:28,283 INFO [StoreOpener-bbb518f4693490310eca052bf28f87fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbb518f4693490310eca052bf28f87fc columnFamilyName family 2024-11-19T07:13:28,283 DEBUG [StoreOpener-bbb518f4693490310eca052bf28f87fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:28,283 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732000407304.82941673e9ad03e5a037091750c3f315., pid=53, masterSystemTime=1732000408159 2024-11-19T07:13:28,284 INFO [StoreOpener-bbb518f4693490310eca052bf28f87fc-1 {}] regionserver.HStore(327): Store=bbb518f4693490310eca052bf28f87fc/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:28,285 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for bbb518f4693490310eca052bf28f87fc 2024-11-19T07:13:28,286 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/bbb518f4693490310eca052bf28f87fc 2024-11-19T07:13:28,286 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=db4ba820395e8b4a2b38d57f8a222e97, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:28,287 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/bbb518f4693490310eca052bf28f87fc 2024-11-19T07:13:28,287 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732000407304.82941673e9ad03e5a037091750c3f315. 2024-11-19T07:13:28,287 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,lll,1732000407304.82941673e9ad03e5a037091750c3f315. 2024-11-19T07:13:28,287 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for bbb518f4693490310eca052bf28f87fc 2024-11-19T07:13:28,287 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,fff,1732000407304.49b850b07eedb76caa94e4c93ac5c832. 2024-11-19T07:13:28,287 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for bbb518f4693490310eca052bf28f87fc 2024-11-19T07:13:28,287 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 49b850b07eedb76caa94e4c93ac5c832, NAME => 'test,fff,1732000407304.49b850b07eedb76caa94e4c93ac5c832.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-19T07:13:28,288 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 49b850b07eedb76caa94e4c93ac5c832 2024-11-19T07:13:28,289 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,fff,1732000407304.49b850b07eedb76caa94e4c93ac5c832.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:28,289 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 49b850b07eedb76caa94e4c93ac5c832 2024-11-19T07:13:28,289 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 49b850b07eedb76caa94e4c93ac5c832 2024-11-19T07:13:28,291 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for bbb518f4693490310eca052bf28f87fc 2024-11-19T07:13:28,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure db4ba820395e8b4a2b38d57f8a222e97, server=db0afee3eab9,42435,1732000404708 because future has completed 2024-11-19T07:13:28,291 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=82941673e9ad03e5a037091750c3f315, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:28,293 INFO [StoreOpener-49b850b07eedb76caa94e4c93ac5c832-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 49b850b07eedb76caa94e4c93ac5c832 2024-11-19T07:13:28,295 INFO [StoreOpener-49b850b07eedb76caa94e4c93ac5c832-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49b850b07eedb76caa94e4c93ac5c832 columnFamilyName family 2024-11-19T07:13:28,296 DEBUG [StoreOpener-49b850b07eedb76caa94e4c93ac5c832-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:28,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82941673e9ad03e5a037091750c3f315, server=db0afee3eab9,34425,1732000404604 because future has completed 2024-11-19T07:13:28,297 INFO [StoreOpener-49b850b07eedb76caa94e4c93ac5c832-1 {}] regionserver.HStore(327): Store=49b850b07eedb76caa94e4c93ac5c832/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:28,297 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 49b850b07eedb76caa94e4c93ac5c832 2024-11-19T07:13:28,297 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/bbb518f4693490310eca052bf28f87fc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T07:13:28,298 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/49b850b07eedb76caa94e4c93ac5c832 2024-11-19T07:13:28,298 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened bbb518f4693490310eca052bf28f87fc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75415769, jitterRate=0.12378253042697906}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:28,299 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bbb518f4693490310eca052bf28f87fc 2024-11-19T07:13:28,299 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for bbb518f4693490310eca052bf28f87fc: Running coprocessor pre-open hook at 1732000408276Writing region info on filesystem at 1732000408276Initializing all the Stores at 1732000408278 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000408278Cleaning up temporary data from old regions at 1732000408287 (+9 ms)Running coprocessor post-open hooks at 1732000408299 (+12 ms)Region opened successfully at 1732000408299 2024-11-19T07:13:28,299 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/49b850b07eedb76caa94e4c93ac5c832 2024-11-19T07:13:28,300 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 49b850b07eedb76caa94e4c93ac5c832 2024-11-19T07:13:28,300 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1732000407304.bbb518f4693490310eca052bf28f87fc., pid=47, masterSystemTime=1732000408092 2024-11-19T07:13:28,300 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 49b850b07eedb76caa94e4c93ac5c832 2024-11-19T07:13:28,302 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 49b850b07eedb76caa94e4c93ac5c832 2024-11-19T07:13:28,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=11 2024-11-19T07:13:28,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure db4ba820395e8b4a2b38d57f8a222e97, server=db0afee3eab9,42435,1732000404708 in 403 msec 2024-11-19T07:13:28,303 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1732000407304.bbb518f4693490310eca052bf28f87fc. 2024-11-19T07:13:28,303 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,kkk,1732000407304.bbb518f4693490310eca052bf28f87fc. 2024-11-19T07:13:28,303 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,yyy,1732000407304.3aa6780c71c61c778946d2086357143d. 2024-11-19T07:13:28,303 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 3aa6780c71c61c778946d2086357143d, NAME => 'test,yyy,1732000407304.3aa6780c71c61c778946d2086357143d.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-19T07:13:28,304 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3aa6780c71c61c778946d2086357143d 2024-11-19T07:13:28,304 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,yyy,1732000407304.3aa6780c71c61c778946d2086357143d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:28,304 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 3aa6780c71c61c778946d2086357143d 2024-11-19T07:13:28,304 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 3aa6780c71c61c778946d2086357143d 2024-11-19T07:13:28,306 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/49b850b07eedb76caa94e4c93ac5c832/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T07:13:28,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=db4ba820395e8b4a2b38d57f8a222e97, ASSIGN in 612 msec 2024-11-19T07:13:28,306 INFO [StoreOpener-3aa6780c71c61c778946d2086357143d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3aa6780c71c61c778946d2086357143d 2024-11-19T07:13:28,307 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 49b850b07eedb76caa94e4c93ac5c832; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72885599, jitterRate=0.08608005940914154}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:28,307 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 49b850b07eedb76caa94e4c93ac5c832 2024-11-19T07:13:28,307 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 49b850b07eedb76caa94e4c93ac5c832: Running coprocessor pre-open hook at 1732000408289Writing region info on filesystem at 1732000408289Initializing all the Stores at 1732000408292 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000408292Cleaning up temporary data from old regions at 1732000408300 (+8 ms)Running coprocessor post-open hooks at 1732000408307 (+7 ms)Region opened successfully at 1732000408307 2024-11-19T07:13:28,308 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=bbb518f4693490310eca052bf28f87fc, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,37853,1732000404756 2024-11-19T07:13:28,309 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1732000407304.49b850b07eedb76caa94e4c93ac5c832., pid=55, masterSystemTime=1732000408159 2024-11-19T07:13:28,309 INFO [StoreOpener-3aa6780c71c61c778946d2086357143d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3aa6780c71c61c778946d2086357143d columnFamilyName family 2024-11-19T07:13:28,309 DEBUG [StoreOpener-3aa6780c71c61c778946d2086357143d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:28,311 INFO [StoreOpener-3aa6780c71c61c778946d2086357143d-1 {}] regionserver.HStore(327): Store=3aa6780c71c61c778946d2086357143d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:28,311 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 3aa6780c71c61c778946d2086357143d 2024-11-19T07:13:28,313 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/3aa6780c71c61c778946d2086357143d 2024-11-19T07:13:28,313 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/3aa6780c71c61c778946d2086357143d 2024-11-19T07:13:28,314 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1732000407304.49b850b07eedb76caa94e4c93ac5c832. 2024-11-19T07:13:28,314 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=db0afee3eab9,37853,1732000404756, table=test, region=bbb518f4693490310eca052bf28f87fc. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T07:13:28,314 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,fff,1732000407304.49b850b07eedb76caa94e4c93ac5c832. 2024-11-19T07:13:28,314 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=db0afee3eab9,37853,1732000404756, table=test, region=08f6487d04261d6f0713b4b5adeacf2e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T07:13:28,314 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,ooo,1732000407304.fdf73eccbee4f01f5a86b87f17dcdb8b. 2024-11-19T07:13:28,314 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 3aa6780c71c61c778946d2086357143d 2024-11-19T07:13:28,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbb518f4693490310eca052bf28f87fc, server=db0afee3eab9,37853,1732000404756 because future has completed 2024-11-19T07:13:28,315 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 3aa6780c71c61c778946d2086357143d 2024-11-19T07:13:28,315 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => fdf73eccbee4f01f5a86b87f17dcdb8b, NAME => 'test,ooo,1732000407304.fdf73eccbee4f01f5a86b87f17dcdb8b.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-19T07:13:28,315 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fdf73eccbee4f01f5a86b87f17dcdb8b 2024-11-19T07:13:28,316 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,ooo,1732000407304.fdf73eccbee4f01f5a86b87f17dcdb8b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:28,316 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for fdf73eccbee4f01f5a86b87f17dcdb8b 2024-11-19T07:13:28,316 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for fdf73eccbee4f01f5a86b87f17dcdb8b 2024-11-19T07:13:28,318 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 3aa6780c71c61c778946d2086357143d 2024-11-19T07:13:28,318 INFO [StoreOpener-fdf73eccbee4f01f5a86b87f17dcdb8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fdf73eccbee4f01f5a86b87f17dcdb8b 2024-11-19T07:13:28,321 INFO [StoreOpener-fdf73eccbee4f01f5a86b87f17dcdb8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fdf73eccbee4f01f5a86b87f17dcdb8b columnFamilyName family 2024-11-19T07:13:28,321 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/3aa6780c71c61c778946d2086357143d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T07:13:28,321 DEBUG [StoreOpener-fdf73eccbee4f01f5a86b87f17dcdb8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:28,322 INFO [StoreOpener-fdf73eccbee4f01f5a86b87f17dcdb8b-1 {}] regionserver.HStore(327): Store=fdf73eccbee4f01f5a86b87f17dcdb8b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:28,322 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 3aa6780c71c61c778946d2086357143d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71305788, jitterRate=0.06253904104232788}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:28,322 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3aa6780c71c61c778946d2086357143d 2024-11-19T07:13:28,322 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for fdf73eccbee4f01f5a86b87f17dcdb8b 2024-11-19T07:13:28,322 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 3aa6780c71c61c778946d2086357143d: Running coprocessor pre-open hook at 1732000408304Writing region info on filesystem at 1732000408304Initializing all the Stores at 1732000408306 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000408306Cleaning up temporary data from old regions at 1732000408315 (+9 ms)Running coprocessor post-open hooks at 1732000408322 (+7 ms)Region opened successfully at 1732000408322 2024-11-19T07:13:28,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=15 2024-11-19T07:13:28,323 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1732000407304.3aa6780c71c61c778946d2086357143d., pid=41, masterSystemTime=1732000408092 2024-11-19T07:13:28,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure bbb518f4693490310eca052bf28f87fc, server=db0afee3eab9,37853,1732000404756 in 326 msec 2024-11-19T07:13:28,324 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/fdf73eccbee4f01f5a86b87f17dcdb8b 2024-11-19T07:13:28,325 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/fdf73eccbee4f01f5a86b87f17dcdb8b 2024-11-19T07:13:28,326 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for fdf73eccbee4f01f5a86b87f17dcdb8b 2024-11-19T07:13:28,326 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for fdf73eccbee4f01f5a86b87f17dcdb8b 2024-11-19T07:13:28,326 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1732000407304.3aa6780c71c61c778946d2086357143d. 2024-11-19T07:13:28,327 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,yyy,1732000407304.3aa6780c71c61c778946d2086357143d. 2024-11-19T07:13:28,327 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,qqq,1732000407304.2df93dc30d9c55d765ada1f50e320789. 2024-11-19T07:13:28,327 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 2df93dc30d9c55d765ada1f50e320789, NAME => 'test,qqq,1732000407304.2df93dc30d9c55d765ada1f50e320789.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-19T07:13:28,327 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2df93dc30d9c55d765ada1f50e320789 2024-11-19T07:13:28,327 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,qqq,1732000407304.2df93dc30d9c55d765ada1f50e320789.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:28,327 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 2df93dc30d9c55d765ada1f50e320789 2024-11-19T07:13:28,328 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 2df93dc30d9c55d765ada1f50e320789 2024-11-19T07:13:28,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN in 633 msec 2024-11-19T07:13:28,330 INFO [StoreOpener-2df93dc30d9c55d765ada1f50e320789-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2df93dc30d9c55d765ada1f50e320789 2024-11-19T07:13:28,330 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for fdf73eccbee4f01f5a86b87f17dcdb8b 2024-11-19T07:13:28,330 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=3aa6780c71c61c778946d2086357143d, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,37853,1732000404756 2024-11-19T07:13:28,332 INFO [StoreOpener-2df93dc30d9c55d765ada1f50e320789-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2df93dc30d9c55d765ada1f50e320789 columnFamilyName family 2024-11-19T07:13:28,332 DEBUG [StoreOpener-2df93dc30d9c55d765ada1f50e320789-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:28,333 INFO [StoreOpener-2df93dc30d9c55d765ada1f50e320789-1 {}] regionserver.HStore(327): Store=2df93dc30d9c55d765ada1f50e320789/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:28,333 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 2df93dc30d9c55d765ada1f50e320789 2024-11-19T07:13:28,334 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/2df93dc30d9c55d765ada1f50e320789 2024-11-19T07:13:28,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3aa6780c71c61c778946d2086357143d, server=db0afee3eab9,37853,1732000404756 because future has completed 2024-11-19T07:13:28,335 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/2df93dc30d9c55d765ada1f50e320789 2024-11-19T07:13:28,336 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 2df93dc30d9c55d765ada1f50e320789 2024-11-19T07:13:28,336 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 2df93dc30d9c55d765ada1f50e320789 2024-11-19T07:13:28,336 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=49b850b07eedb76caa94e4c93ac5c832, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:28,337 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/fdf73eccbee4f01f5a86b87f17dcdb8b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T07:13:28,338 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 2df93dc30d9c55d765ada1f50e320789 2024-11-19T07:13:28,338 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened fdf73eccbee4f01f5a86b87f17dcdb8b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66690122, jitterRate=-0.006239742040634155}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:28,339 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fdf73eccbee4f01f5a86b87f17dcdb8b 2024-11-19T07:13:28,339 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for fdf73eccbee4f01f5a86b87f17dcdb8b: Running coprocessor pre-open hook at 1732000408316Writing region info on filesystem at 1732000408316Initializing all the Stores at 1732000408318 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000408318Cleaning up temporary data from old regions at 1732000408326 (+8 ms)Running coprocessor post-open hooks at 1732000408339 (+13 ms)Region opened successfully at 1732000408339 2024-11-19T07:13:28,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49b850b07eedb76caa94e4c93ac5c832, server=db0afee3eab9,34425,1732000404604 because future has completed 2024-11-19T07:13:28,341 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1732000407304.fdf73eccbee4f01f5a86b87f17dcdb8b., pid=50, masterSystemTime=1732000408159 2024-11-19T07:13:28,343 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/2df93dc30d9c55d765ada1f50e320789/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T07:13:28,344 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 2df93dc30d9c55d765ada1f50e320789; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62580447, jitterRate=-0.06747867166996002}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:28,344 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1732000407304.fdf73eccbee4f01f5a86b87f17dcdb8b. 2024-11-19T07:13:28,344 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2df93dc30d9c55d765ada1f50e320789 2024-11-19T07:13:28,344 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,ooo,1732000407304.fdf73eccbee4f01f5a86b87f17dcdb8b. 2024-11-19T07:13:28,344 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 2df93dc30d9c55d765ada1f50e320789: Running coprocessor pre-open hook at 1732000408328Writing region info on filesystem at 1732000408328Initializing all the Stores at 1732000408329 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000408329Cleaning up temporary data from old regions at 1732000408336 (+7 ms)Running coprocessor post-open hooks at 1732000408344 (+8 ms)Region opened successfully at 1732000408344 2024-11-19T07:13:28,344 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,uuu,1732000407304.9357ae6c6e8337f25dacbf0a10701ef6. 2024-11-19T07:13:28,344 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 9357ae6c6e8337f25dacbf0a10701ef6, NAME => 'test,uuu,1732000407304.9357ae6c6e8337f25dacbf0a10701ef6.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-19T07:13:28,345 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9357ae6c6e8337f25dacbf0a10701ef6 2024-11-19T07:13:28,345 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,uuu,1732000407304.9357ae6c6e8337f25dacbf0a10701ef6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:28,345 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 9357ae6c6e8337f25dacbf0a10701ef6 2024-11-19T07:13:28,345 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 9357ae6c6e8337f25dacbf0a10701ef6 2024-11-19T07:13:28,346 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1732000407304.2df93dc30d9c55d765ada1f50e320789., pid=43, masterSystemTime=1732000408092 2024-11-19T07:13:28,348 INFO [StoreOpener-9357ae6c6e8337f25dacbf0a10701ef6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9357ae6c6e8337f25dacbf0a10701ef6 2024-11-19T07:13:28,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=10 2024-11-19T07:13:28,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 49b850b07eedb76caa94e4c93ac5c832, server=db0afee3eab9,34425,1732000404604 in 300 msec 2024-11-19T07:13:28,349 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1732000407304.2df93dc30d9c55d765ada1f50e320789. 2024-11-19T07:13:28,349 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,qqq,1732000407304.2df93dc30d9c55d765ada1f50e320789. 2024-11-19T07:13:28,349 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,nnn,1732000407304.74657be6232ad22aaca548d82491de41. 2024-11-19T07:13:28,349 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 74657be6232ad22aaca548d82491de41, NAME => 'test,nnn,1732000407304.74657be6232ad22aaca548d82491de41.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-19T07:13:28,350 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 74657be6232ad22aaca548d82491de41 2024-11-19T07:13:28,350 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,nnn,1732000407304.74657be6232ad22aaca548d82491de41.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:28,350 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 74657be6232ad22aaca548d82491de41 2024-11-19T07:13:28,350 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 74657be6232ad22aaca548d82491de41 2024-11-19T07:13:28,350 INFO [StoreOpener-9357ae6c6e8337f25dacbf0a10701ef6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9357ae6c6e8337f25dacbf0a10701ef6 columnFamilyName family 2024-11-19T07:13:28,351 DEBUG [StoreOpener-9357ae6c6e8337f25dacbf0a10701ef6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:28,351 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=2df93dc30d9c55d765ada1f50e320789, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,37853,1732000404756 2024-11-19T07:13:28,351 INFO [StoreOpener-9357ae6c6e8337f25dacbf0a10701ef6-1 {}] regionserver.HStore(327): Store=9357ae6c6e8337f25dacbf0a10701ef6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:28,351 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 9357ae6c6e8337f25dacbf0a10701ef6 2024-11-19T07:13:28,352 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/9357ae6c6e8337f25dacbf0a10701ef6 2024-11-19T07:13:28,353 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/9357ae6c6e8337f25dacbf0a10701ef6 2024-11-19T07:13:28,354 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 9357ae6c6e8337f25dacbf0a10701ef6 2024-11-19T07:13:28,354 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 9357ae6c6e8337f25dacbf0a10701ef6 2024-11-19T07:13:28,354 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2df93dc30d9c55d765ada1f50e320789, server=db0afee3eab9,37853,1732000404756 because future has completed 2024-11-19T07:13:28,355 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 9357ae6c6e8337f25dacbf0a10701ef6 2024-11-19T07:13:28,356 INFO [StoreOpener-74657be6232ad22aaca548d82491de41-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 74657be6232ad22aaca548d82491de41 2024-11-19T07:13:28,358 INFO [StoreOpener-74657be6232ad22aaca548d82491de41-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 74657be6232ad22aaca548d82491de41 columnFamilyName family 2024-11-19T07:13:28,358 DEBUG [StoreOpener-74657be6232ad22aaca548d82491de41-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:28,359 INFO [StoreOpener-74657be6232ad22aaca548d82491de41-1 {}] regionserver.HStore(327): Store=74657be6232ad22aaca548d82491de41/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:28,359 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/9357ae6c6e8337f25dacbf0a10701ef6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T07:13:28,359 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 74657be6232ad22aaca548d82491de41 2024-11-19T07:13:28,360 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 9357ae6c6e8337f25dacbf0a10701ef6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67241818, jitterRate=0.0019811689853668213}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:28,360 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9357ae6c6e8337f25dacbf0a10701ef6 2024-11-19T07:13:28,360 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/74657be6232ad22aaca548d82491de41 2024-11-19T07:13:28,360 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 9357ae6c6e8337f25dacbf0a10701ef6: Running coprocessor pre-open hook at 1732000408345Writing region info on filesystem at 1732000408345Initializing all the Stores at 1732000408347 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000408347Cleaning up temporary data from old regions at 1732000408354 (+7 ms)Running coprocessor post-open hooks at 1732000408360 (+6 ms)Region opened successfully at 1732000408360 2024-11-19T07:13:28,361 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/74657be6232ad22aaca548d82491de41 2024-11-19T07:13:28,361 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 74657be6232ad22aaca548d82491de41 2024-11-19T07:13:28,361 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 74657be6232ad22aaca548d82491de41 2024-11-19T07:13:28,361 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1732000407304.9357ae6c6e8337f25dacbf0a10701ef6., pid=52, masterSystemTime=1732000408159 2024-11-19T07:13:28,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=21 2024-11-19T07:13:28,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 2df93dc30d9c55d765ada1f50e320789, server=db0afee3eab9,37853,1732000404756 in 389 msec 2024-11-19T07:13:28,364 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 74657be6232ad22aaca548d82491de41 2024-11-19T07:13:28,364 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1732000407304.9357ae6c6e8337f25dacbf0a10701ef6. 2024-11-19T07:13:28,364 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,uuu,1732000407304.9357ae6c6e8337f25dacbf0a10701ef6. 2024-11-19T07:13:28,364 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,rrr,1732000407304.dba97405d3b35647d42d266df5e81cec. 2024-11-19T07:13:28,364 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => dba97405d3b35647d42d266df5e81cec, NAME => 'test,rrr,1732000407304.dba97405d3b35647d42d266df5e81cec.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-19T07:13:28,365 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test dba97405d3b35647d42d266df5e81cec 2024-11-19T07:13:28,365 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,rrr,1732000407304.dba97405d3b35647d42d266df5e81cec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:28,365 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for dba97405d3b35647d42d266df5e81cec 2024-11-19T07:13:28,365 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=9357ae6c6e8337f25dacbf0a10701ef6, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:28,365 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for dba97405d3b35647d42d266df5e81cec 2024-11-19T07:13:28,368 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/74657be6232ad22aaca548d82491de41/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T07:13:28,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9357ae6c6e8337f25dacbf0a10701ef6, server=db0afee3eab9,34425,1732000404604 because future has completed 2024-11-19T07:13:28,369 INFO [StoreOpener-dba97405d3b35647d42d266df5e81cec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region dba97405d3b35647d42d266df5e81cec 2024-11-19T07:13:28,369 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 74657be6232ad22aaca548d82491de41; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60996032, jitterRate=-0.09108829498291016}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:28,369 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 74657be6232ad22aaca548d82491de41 2024-11-19T07:13:28,370 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 74657be6232ad22aaca548d82491de41: Running coprocessor pre-open hook at 1732000408350Writing region info on filesystem at 1732000408350Initializing all the Stores at 1732000408352 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000408352Cleaning up temporary data from old regions at 1732000408361 (+9 ms)Running coprocessor post-open hooks at 1732000408369 (+8 ms)Region opened successfully at 1732000408369 2024-11-19T07:13:28,371 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1732000407304.74657be6232ad22aaca548d82491de41., pid=48, masterSystemTime=1732000408092 2024-11-19T07:13:28,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN in 677 msec 2024-11-19T07:13:28,372 INFO [StoreOpener-dba97405d3b35647d42d266df5e81cec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dba97405d3b35647d42d266df5e81cec columnFamilyName family 2024-11-19T07:13:28,372 DEBUG [StoreOpener-dba97405d3b35647d42d266df5e81cec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:28,373 INFO [StoreOpener-dba97405d3b35647d42d266df5e81cec-1 {}] regionserver.HStore(327): Store=dba97405d3b35647d42d266df5e81cec/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:28,373 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for dba97405d3b35647d42d266df5e81cec 2024-11-19T07:13:28,374 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1732000407304.74657be6232ad22aaca548d82491de41. 2024-11-19T07:13:28,374 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,nnn,1732000407304.74657be6232ad22aaca548d82491de41. 2024-11-19T07:13:28,375 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/dba97405d3b35647d42d266df5e81cec 2024-11-19T07:13:28,376 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/dba97405d3b35647d42d266df5e81cec 2024-11-19T07:13:28,376 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for dba97405d3b35647d42d266df5e81cec 2024-11-19T07:13:28,376 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for dba97405d3b35647d42d266df5e81cec 2024-11-19T07:13:28,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=25 2024-11-19T07:13:28,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 9357ae6c6e8337f25dacbf0a10701ef6, server=db0afee3eab9,34425,1732000404604 in 349 msec 2024-11-19T07:13:28,378 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=db0afee3eab9,34425,1732000404604, table=test, region=12dcc969f79695b5f0d224b605268d2d. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T07:13:28,378 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=db0afee3eab9,34425,1732000404604, table=test, region=82941673e9ad03e5a037091750c3f315. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T07:13:28,380 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for dba97405d3b35647d42d266df5e81cec 2024-11-19T07:13:28,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9357ae6c6e8337f25dacbf0a10701ef6, ASSIGN in 686 msec 2024-11-19T07:13:28,383 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/dba97405d3b35647d42d266df5e81cec/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T07:13:28,385 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=74657be6232ad22aaca548d82491de41, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,37853,1732000404756 2024-11-19T07:13:28,385 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened dba97405d3b35647d42d266df5e81cec; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59169615, jitterRate=-0.11830402910709381}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:28,385 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dba97405d3b35647d42d266df5e81cec 2024-11-19T07:13:28,385 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for dba97405d3b35647d42d266df5e81cec: Running coprocessor pre-open hook at 1732000408365Writing region info on filesystem at 1732000408365Initializing all the Stores at 1732000408367 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000408368 (+1 ms)Cleaning up temporary data from old regions at 1732000408376 (+8 ms)Running coprocessor post-open hooks at 1732000408385 (+9 ms)Region opened successfully at 1732000408385 2024-11-19T07:13:28,388 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732000407304.dba97405d3b35647d42d266df5e81cec., pid=51, masterSystemTime=1732000408159 2024-11-19T07:13:28,389 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 74657be6232ad22aaca548d82491de41, server=db0afee3eab9,37853,1732000404756 because future has completed 2024-11-19T07:13:28,391 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=49b850b07eedb76caa94e4c93ac5c832, ASSIGN in 697 msec 2024-11-19T07:13:28,393 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732000407304.dba97405d3b35647d42d266df5e81cec. 2024-11-19T07:13:28,393 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,rrr,1732000407304.dba97405d3b35647d42d266df5e81cec. 2024-11-19T07:13:28,393 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=db0afee3eab9,42435,1732000404708, table=test, region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T07:13:28,393 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,iii,1732000407304.43676ec232db06c9bf4af64db1ede4f5. 2024-11-19T07:13:28,393 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 43676ec232db06c9bf4af64db1ede4f5, NAME => 'test,iii,1732000407304.43676ec232db06c9bf4af64db1ede4f5.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-19T07:13:28,394 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 43676ec232db06c9bf4af64db1ede4f5 2024-11-19T07:13:28,394 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,iii,1732000407304.43676ec232db06c9bf4af64db1ede4f5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:28,394 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 43676ec232db06c9bf4af64db1ede4f5 2024-11-19T07:13:28,394 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 43676ec232db06c9bf4af64db1ede4f5 2024-11-19T07:13:28,396 INFO [StoreOpener-43676ec232db06c9bf4af64db1ede4f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 43676ec232db06c9bf4af64db1ede4f5 2024-11-19T07:13:28,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=18 2024-11-19T07:13:28,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 74657be6232ad22aaca548d82491de41, server=db0afee3eab9,37853,1732000404756 in 396 msec 2024-11-19T07:13:28,399 INFO [StoreOpener-43676ec232db06c9bf4af64db1ede4f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43676ec232db06c9bf4af64db1ede4f5 columnFamilyName family 2024-11-19T07:13:28,399 DEBUG [StoreOpener-43676ec232db06c9bf4af64db1ede4f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:28,400 INFO [StoreOpener-43676ec232db06c9bf4af64db1ede4f5-1 {}] regionserver.HStore(327): Store=43676ec232db06c9bf4af64db1ede4f5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:28,400 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 43676ec232db06c9bf4af64db1ede4f5 2024-11-19T07:13:28,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, ASSIGN in 706 msec 2024-11-19T07:13:28,401 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/43676ec232db06c9bf4af64db1ede4f5 2024-11-19T07:13:28,402 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/43676ec232db06c9bf4af64db1ede4f5 2024-11-19T07:13:28,402 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=dba97405d3b35647d42d266df5e81cec, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:28,402 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 43676ec232db06c9bf4af64db1ede4f5 2024-11-19T07:13:28,402 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 43676ec232db06c9bf4af64db1ede4f5 2024-11-19T07:13:28,405 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure dba97405d3b35647d42d266df5e81cec, server=db0afee3eab9,34425,1732000404604 because future has completed 2024-11-19T07:13:28,406 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 43676ec232db06c9bf4af64db1ede4f5 2024-11-19T07:13:28,406 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=fdf73eccbee4f01f5a86b87f17dcdb8b, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:28,409 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/43676ec232db06c9bf4af64db1ede4f5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T07:13:28,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure fdf73eccbee4f01f5a86b87f17dcdb8b, server=db0afee3eab9,34425,1732000404604 because future has completed 2024-11-19T07:13:28,410 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 43676ec232db06c9bf4af64db1ede4f5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69434881, jitterRate=0.034660354256629944}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:28,410 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 43676ec232db06c9bf4af64db1ede4f5 2024-11-19T07:13:28,410 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 43676ec232db06c9bf4af64db1ede4f5: Running coprocessor pre-open hook at 1732000408394Writing region info on filesystem at 1732000408394Initializing all the Stores at 1732000408396 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000408396Cleaning up temporary data from old regions at 1732000408402 (+6 ms)Running coprocessor post-open hooks at 1732000408410 (+8 ms)Region opened successfully at 1732000408410 2024-11-19T07:13:28,411 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1732000407304.43676ec232db06c9bf4af64db1ede4f5., pid=56, masterSystemTime=1732000408159 2024-11-19T07:13:28,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=22 2024-11-19T07:13:28,414 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure dba97405d3b35647d42d266df5e81cec, server=db0afee3eab9,34425,1732000404604 in 393 msec 2024-11-19T07:13:28,414 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1732000407304.43676ec232db06c9bf4af64db1ede4f5. 2024-11-19T07:13:28,414 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,iii,1732000407304.43676ec232db06c9bf4af64db1ede4f5. 2024-11-19T07:13:28,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dba97405d3b35647d42d266df5e81cec, ASSIGN in 723 msec 2024-11-19T07:13:28,418 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=43676ec232db06c9bf4af64db1ede4f5, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:28,419 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=db0afee3eab9,37853,1732000404756, table=test, region=08f6487d04261d6f0713b4b5adeacf2e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T07:13:28,419 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=db0afee3eab9,37853,1732000404756, table=test, region=3aa6780c71c61c778946d2086357143d. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T07:13:28,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43676ec232db06c9bf4af64db1ede4f5, server=db0afee3eab9,34425,1732000404604 because future has completed 2024-11-19T07:13:28,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=19 2024-11-19T07:13:28,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure fdf73eccbee4f01f5a86b87f17dcdb8b, server=db0afee3eab9,34425,1732000404604 in 411 msec 2024-11-19T07:13:28,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fdf73eccbee4f01f5a86b87f17dcdb8b, ASSIGN in 734 msec 2024-11-19T07:13:28,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=13 2024-11-19T07:13:28,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 43676ec232db06c9bf4af64db1ede4f5, server=db0afee3eab9,34425,1732000404604 in 383 msec 2024-11-19T07:13:28,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=43676ec232db06c9bf4af64db1ede4f5, ASSIGN in 741 msec 2024-11-19T07:13:28,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=29 2024-11-19T07:13:28,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 3aa6780c71c61c778946d2086357143d, server=db0afee3eab9,37853,1732000404756 in 486 msec 2024-11-19T07:13:28,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN in 748 msec 2024-11-19T07:13:28,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=16 2024-11-19T07:13:28,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 82941673e9ad03e5a037091750c3f315, server=db0afee3eab9,34425,1732000404604 in 410 msec 2024-11-19T07:13:28,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=82941673e9ad03e5a037091750c3f315, ASSIGN in 754 msec 2024-11-19T07:13:28,448 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=66fe58970f9c4a1cba9c3e3da843ed3d, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,37853,1732000404756 2024-11-19T07:13:28,451 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66fe58970f9c4a1cba9c3e3da843ed3d, server=db0afee3eab9,37853,1732000404756 because future has completed 2024-11-19T07:13:28,454 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=6 2024-11-19T07:13:28,454 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 12dcc969f79695b5f0d224b605268d2d, server=db0afee3eab9,34425,1732000404604 in 414 msec 2024-11-19T07:13:28,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=12dcc969f79695b5f0d224b605268d2d, ASSIGN in 763 msec 2024-11-19T07:13:28,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=12 2024-11-19T07:13:28,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 66fe58970f9c4a1cba9c3e3da843ed3d, server=db0afee3eab9,37853,1732000404756 in 484 msec 2024-11-19T07:13:28,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN in 768 msec 2024-11-19T07:13:28,463 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=2f38687f659a16d9e2a781f2a75d514a, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,37853,1732000404756 2024-11-19T07:13:28,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,37853,1732000404756 because future has completed 2024-11-19T07:13:28,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=17 2024-11-19T07:13:28,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure d2ef4078aa4f5f4b6f4dc7188dd2f1cb, server=db0afee3eab9,42435,1732000404708 in 566 msec 2024-11-19T07:13:28,470 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, ASSIGN in 777 msec 2024-11-19T07:13:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T07:13:28,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=24 2024-11-19T07:13:28,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,37853,1732000404756 in 532 msec 2024-11-19T07:13:28,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN in 783 msec 2024-11-19T07:13:28,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=26 2024-11-19T07:13:28,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 08f6487d04261d6f0713b4b5adeacf2e, server=db0afee3eab9,37853,1732000404756 in 533 msec 2024-11-19T07:13:28,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=4 2024-11-19T07:13:28,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN in 789 msec 2024-11-19T07:13:28,485 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T07:13:28,485 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732000408485"}]},"ts":"1732000408485"} 2024-11-19T07:13:28,488 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-19T07:13:28,489 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T07:13:28,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.1780 sec 2024-11-19T07:13:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T07:13:29,483 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-19T07:13:29,483 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-19T07:13:29,485 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T07:13:29,504 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-19T07:13:29,505 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T07:13:29,506 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-19T07:13:29,506 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T07:13:29,513 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-19T07:13:29,538 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-19T07:13:29,556 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T07:13:29,558 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T07:13:29,560 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T07:13:29,564 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T07:13:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 12dcc969f79695b5f0d224b605268d2d changed from -1.0 to 0.0, refreshing cache 2024-11-19T07:13:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 49b850b07eedb76caa94e4c93ac5c832 changed from -1.0 to 0.0, refreshing cache 2024-11-19T07:13:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 43676ec232db06c9bf4af64db1ede4f5 changed from -1.0 to 0.0, refreshing cache 2024-11-19T07:13:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 82941673e9ad03e5a037091750c3f315 changed from -1.0 to 0.0, refreshing cache 2024-11-19T07:13:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fdf73eccbee4f01f5a86b87f17dcdb8b changed from -1.0 to 0.0, refreshing cache 2024-11-19T07:13:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region dba97405d3b35647d42d266df5e81cec changed from -1.0 to 0.0, refreshing cache 2024-11-19T07:13:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9357ae6c6e8337f25dacbf0a10701ef6 changed from -1.0 to 0.0, refreshing cache 2024-11-19T07:13:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 126b77a274d2ba63b12dd23c9e7ababb changed from -1.0 to 0.0, refreshing cache 2024-11-19T07:13:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7d9d31e1a2abc752a2056ed0e3a3fcec changed from -1.0 to 0.0, refreshing cache 2024-11-19T07:13:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e880c03f4c433baa6242dd65ef2d8486 changed from -1.0 to 0.0, refreshing cache 2024-11-19T07:13:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 66fe58970f9c4a1cba9c3e3da843ed3d changed from -1.0 to 0.0, refreshing cache 2024-11-19T07:13:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region bbb518f4693490310eca052bf28f87fc changed from -1.0 to 0.0, refreshing cache 2024-11-19T07:13:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 74657be6232ad22aaca548d82491de41 changed from -1.0 to 0.0, refreshing cache 2024-11-19T07:13:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2df93dc30d9c55d765ada1f50e320789 changed from -1.0 to 0.0, refreshing cache 2024-11-19T07:13:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2f38687f659a16d9e2a781f2a75d514a changed from -1.0 to 0.0, refreshing cache 2024-11-19T07:13:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 08f6487d04261d6f0713b4b5adeacf2e changed from -1.0 to 0.0, refreshing cache 2024-11-19T07:13:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3aa6780c71c61c778946d2086357143d changed from -1.0 to 0.0, refreshing cache 2024-11-19T07:13:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 02309bab61f6207d73b9f19985e1fdb9 changed from -1.0 to 0.0, refreshing cache 2024-11-19T07:13:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1ece759217c055d9a295034ce69dd86d changed from -1.0 to 0.0, refreshing cache 2024-11-19T07:13:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region db4ba820395e8b4a2b38d57f8a222e97 changed from -1.0 to 0.0, refreshing cache 2024-11-19T07:13:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4421c1189c5fdff3a40dcc58728c41ab changed from -1.0 to 0.0, refreshing cache 2024-11-19T07:13:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d2ef4078aa4f5f4b6f4dc7188dd2f1cb changed from -1.0 to 0.0, refreshing cache 2024-11-19T07:13:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3a637105eb1d8b75696b3cfabe5d8173 changed from -1.0 to 0.0, refreshing cache 2024-11-19T07:13:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d703151d82f3a429f348967988283542 changed from -1.0 to 0.0, refreshing cache 2024-11-19T07:13:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 611b07fbb4fb5e3ea7255d9fa0f9fe40 changed from -1.0 to 0.0, refreshing cache 2024-11-19T07:13:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 32d987cca115e6f476cdb5686be2b726 changed from -1.0 to 0.0, refreshing cache 2024-11-19T07:13:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-19T07:13:29,630 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-19T07:13:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-19T07:13:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.BalancerClusterState(204): Hosts are {db0afee3eab9=0} racks are {/default-rack=0} 2024-11-19T07:13:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.BalancerClusterState(303): server 0 has 8 regions 2024-11-19T07:13:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.BalancerClusterState(303): server 1 has 10 regions 2024-11-19T07:13:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-19T07:13:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-19T07:13:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-19T07:13:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-19T07:13:29,631 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-19T07:13:29,631 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-19T07:13:29,631 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-19T07:13:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-19T07:13:29,636 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.StochasticLoadBalancer(403): Cluster wide - Calculating plan. may take up to 30000ms to complete. 2024-11-19T07:13:29,637 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.StochasticLoadBalancer(515): Start StochasticLoadBalancer.balancer, initial weighted average imbalance=0.06931633435746164, functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.06415002990995843, need balance); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); computedMaxSteps=64800 2024-11-19T07:13:30,122 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] balancer.StochasticLoadBalancer(562): Finished computing new moving plan. Computation took 490 ms to try 64800 different iterations. Found a solution that moves 1 regions; Going from a computed imbalance of 0.06931633435746164 to a new imbalance of 0.01690158032826896. funtionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.037037037037037035, need balance); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-19T07:13:30,125 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] master.HMaster(2167): Balancer plans size is 1, the balance interval is 300000 ms, and the max number regions in transition is 27 2024-11-19T07:13:30,125 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] master.HMaster(2172): balance hri=74657be6232ad22aaca548d82491de41, source=db0afee3eab9,37853,1732000404756, destination=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:30,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, REOPEN/MOVE 2024-11-19T07:13:30,127 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, REOPEN/MOVE 2024-11-19T07:13:30,129 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=74657be6232ad22aaca548d82491de41, regionState=CLOSING, regionLocation=db0afee3eab9,37853,1732000404756 2024-11-19T07:13:30,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, REOPEN/MOVE because future has completed 2024-11-19T07:13:30,132 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T07:13:30,132 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 74657be6232ad22aaca548d82491de41, server=db0afee3eab9,37853,1732000404756}] 2024-11-19T07:13:30,289 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(122): Close 74657be6232ad22aaca548d82491de41 2024-11-19T07:13:30,289 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T07:13:30,290 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1722): Closing 74657be6232ad22aaca548d82491de41, disabling compactions & flushes 2024-11-19T07:13:30,290 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1755): Closing region test,nnn,1732000407304.74657be6232ad22aaca548d82491de41. 2024-11-19T07:13:30,290 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732000407304.74657be6232ad22aaca548d82491de41. 2024-11-19T07:13:30,290 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732000407304.74657be6232ad22aaca548d82491de41. after waiting 0 ms 2024-11-19T07:13:30,290 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732000407304.74657be6232ad22aaca548d82491de41. 2024-11-19T07:13:30,295 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/74657be6232ad22aaca548d82491de41/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T07:13:30,298 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1973): Closed test,nnn,1732000407304.74657be6232ad22aaca548d82491de41. 2024-11-19T07:13:30,298 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1676): Region close journal for 74657be6232ad22aaca548d82491de41: Waiting for close lock at 1732000410290Running coprocessor pre-close hooks at 1732000410290Disabling compacts and flushes for region at 1732000410290Disabling writes for close at 1732000410290Writing region close event to WAL at 1732000410291 (+1 ms)Running coprocessor post-close hooks at 1732000410296 (+5 ms)Closed at 1732000410297 (+1 ms) 2024-11-19T07:13:30,298 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionServer(3302): Adding 74657be6232ad22aaca548d82491de41 move to db0afee3eab9,34425,1732000404604 record at close sequenceid=2 2024-11-19T07:13:30,301 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(157): Closed 74657be6232ad22aaca548d82491de41 2024-11-19T07:13:30,302 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=74657be6232ad22aaca548d82491de41, regionState=CLOSED 2024-11-19T07:13:30,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 74657be6232ad22aaca548d82491de41, server=db0afee3eab9,37853,1732000404756 because future has completed 2024-11-19T07:13:30,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-19T07:13:30,308 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; CloseRegionProcedure 74657be6232ad22aaca548d82491de41, server=db0afee3eab9,37853,1732000404756 in 173 msec 2024-11-19T07:13:30,308 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, REOPEN/MOVE; state=CLOSED, location=db0afee3eab9,34425,1732000404604; forceNewPlan=false, retain=false 2024-11-19T07:13:30,459 INFO [db0afee3eab9:43779 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-19T07:13:30,460 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=74657be6232ad22aaca548d82491de41, regionState=OPENING, regionLocation=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:30,463 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, REOPEN/MOVE because future has completed 2024-11-19T07:13:30,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 74657be6232ad22aaca548d82491de41, server=db0afee3eab9,34425,1732000404604}] 2024-11-19T07:13:30,627 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open test,nnn,1732000407304.74657be6232ad22aaca548d82491de41. 2024-11-19T07:13:30,627 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 74657be6232ad22aaca548d82491de41, NAME => 'test,nnn,1732000407304.74657be6232ad22aaca548d82491de41.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-19T07:13:30,628 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 74657be6232ad22aaca548d82491de41 2024-11-19T07:13:30,628 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated test,nnn,1732000407304.74657be6232ad22aaca548d82491de41.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:30,628 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for 74657be6232ad22aaca548d82491de41 2024-11-19T07:13:30,628 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for 74657be6232ad22aaca548d82491de41 2024-11-19T07:13:30,631 INFO [StoreOpener-74657be6232ad22aaca548d82491de41-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 74657be6232ad22aaca548d82491de41 2024-11-19T07:13:30,632 INFO [StoreOpener-74657be6232ad22aaca548d82491de41-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 74657be6232ad22aaca548d82491de41 columnFamilyName family 2024-11-19T07:13:30,632 DEBUG [StoreOpener-74657be6232ad22aaca548d82491de41-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:30,633 INFO [StoreOpener-74657be6232ad22aaca548d82491de41-1 {}] regionserver.HStore(327): Store=74657be6232ad22aaca548d82491de41/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:30,633 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for 74657be6232ad22aaca548d82491de41 2024-11-19T07:13:30,634 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/74657be6232ad22aaca548d82491de41 2024-11-19T07:13:30,636 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/74657be6232ad22aaca548d82491de41 2024-11-19T07:13:30,636 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for 74657be6232ad22aaca548d82491de41 2024-11-19T07:13:30,636 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 74657be6232ad22aaca548d82491de41 2024-11-19T07:13:30,639 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for 74657be6232ad22aaca548d82491de41 2024-11-19T07:13:30,640 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened 74657be6232ad22aaca548d82491de41; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70586709, jitterRate=0.0518239289522171}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:30,640 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 74657be6232ad22aaca548d82491de41 2024-11-19T07:13:30,640 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for 74657be6232ad22aaca548d82491de41: Running coprocessor pre-open hook at 1732000410629Writing region info on filesystem at 1732000410629Initializing all the Stores at 1732000410630 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000410630Cleaning up temporary data from old regions at 1732000410636 (+6 ms)Running coprocessor post-open hooks at 1732000410640 (+4 ms)Region opened successfully at 1732000410640 2024-11-19T07:13:30,641 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1732000407304.74657be6232ad22aaca548d82491de41., pid=59, masterSystemTime=1732000410617 2024-11-19T07:13:30,644 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1732000407304.74657be6232ad22aaca548d82491de41. 2024-11-19T07:13:30,644 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened test,nnn,1732000407304.74657be6232ad22aaca548d82491de41. 2024-11-19T07:13:30,645 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=74657be6232ad22aaca548d82491de41, regionState=OPEN, openSeqNum=5, regionLocation=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:30,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 74657be6232ad22aaca548d82491de41, server=db0afee3eab9,34425,1732000404604 because future has completed 2024-11-19T07:13:30,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=57 2024-11-19T07:13:30,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=57, state=SUCCESS, hasLock=false; OpenRegionProcedure 74657be6232ad22aaca548d82491de41, server=db0afee3eab9,34425,1732000404604 in 185 msec 2024-11-19T07:13:30,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, REOPEN/MOVE in 526 msec 2024-11-19T07:13:30,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-19T07:13:30,731 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T07:13:30,741 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-19T07:13:30,757 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=386, OpenFileDescriptor=647, MaxFileDescriptor=1048576, SystemLoadAverage=336, ProcessCount=11, AvailableMemoryMB=12301 2024-11-19T07:13:30,760 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-19T07:13:30,761 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-19T07:13:30,761 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing db0afee3eab9,37853,1732000404756 2024-11-19T07:13:30,762 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server db0afee3eab9,37853,1732000404756: Simulated kill ***** 2024-11-19T07:13:30,762 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-19T07:13:30,764 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-19T07:13:30,766 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-19T07:13:30,766 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-19T07:13:30,767 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-19T07:13:30,767 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 342746808 }, "NonHeapMemoryUsage": { "committed": 153747456, "init": 7667712, "max": -1, "used": 151329816 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "db0afee3eab9" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "db0afee3eab9" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "db0afee3eab9" } ] } 2024-11-19T07:13:30,769 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43779 {}] master.MasterRpcServices(700): db0afee3eab9,37853,1732000404756 reported a fatal error: ***** ABORTING region server db0afee3eab9,37853,1732000404756: Simulated kill ***** 2024-11-19T07:13:30,771 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'db0afee3eab9,37853,1732000404756' ***** 2024-11-19T07:13:30,771 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-19T07:13:30,771 INFO [RS:2;db0afee3eab9:37853 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T07:13:30,772 INFO [RS:2;db0afee3eab9:37853 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-19T07:13:30,772 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T07:13:30,772 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T07:13:30,772 INFO [RS:2;db0afee3eab9:37853 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-19T07:13:30,772 INFO [RS:2;db0afee3eab9:37853 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T07:13:30,772 INFO [RS:2;db0afee3eab9:37853 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;db0afee3eab9:37853. 2024-11-19T07:13:30,772 DEBUG [RS:2;db0afee3eab9:37853 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T07:13:30,776 DEBUG [RS:2;db0afee3eab9:37853 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T07:13:30,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741836_1012 (size=33749) 2024-11-19T07:13:30,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741836_1012 (size=33749) 2024-11-19T07:13:30,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741836_1012 (size=33749) 2024-11-19T07:13:30,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741835_1011 (size=2774) 2024-11-19T07:13:30,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741835_1011 (size=2774) 2024-11-19T07:13:30,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741835_1011 (size=2774) 2024-11-19T07:13:30,788 DEBUG [RS:2;db0afee3eab9:37853 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T07:13:30,788 INFO [RS:2;db0afee3eab9:37853 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T07:13:30,789 INFO [RS:2;db0afee3eab9:37853 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37853 2024-11-19T07:13:30,804 INFO [regionserver/db0afee3eab9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T07:13:30,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T07:13:30,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x10151f0c9da0003, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/db0afee3eab9,37853,1732000404756 2024-11-19T07:13:30,849 INFO [RS:2;db0afee3eab9:37853 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T07:13:30,850 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [db0afee3eab9,37853,1732000404756] 2024-11-19T07:13:30,869 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/db0afee3eab9,37853,1732000404756 already deleted, retry=false 2024-11-19T07:13:30,870 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of db0afee3eab9,37853,1732000404756 on db0afee3eab9,43779,1732000403834 2024-11-19T07:13:30,876 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true 2024-11-19T07:13:30,880 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=60 for db0afee3eab9,37853,1732000404756 (carryingMeta=true) db0afee3eab9,37853,1732000404756/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@6ce6ced1[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-19T07:13:30,881 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true 2024-11-19T07:13:30,882 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true, isMeta: true 2024-11-19T07:13:30,884 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756-splitting 2024-11-19T07:13:30,885 INFO [PEWorker-1 {}] master.SplitWALManager(105): db0afee3eab9,37853,1732000404756 WAL count=1, meta=true 2024-11-19T07:13:30,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta}] 2024-11-19T07:13:30,894 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:30,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604}] 2024-11-19T07:13:30,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x10151f0c9da0003, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T07:13:30,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x10151f0c9da0003, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T07:13:30,960 INFO [RS:2;db0afee3eab9:37853 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T07:13:30,960 INFO [RS:2;db0afee3eab9:37853 {}] regionserver.HRegionServer(1031): Exiting; stopping=db0afee3eab9,37853,1732000404756; zookeeper connection closed. 2024-11-19T07:13:30,961 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@b543331 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@b543331 2024-11-19T07:13:30,973 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T07:13:31,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=62 2024-11-19T07:13:31,091 INFO [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(299): Splitting hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756-splitting/db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, size=33.0 K (33749bytes) 2024-11-19T07:13:31,091 DEBUG [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-19T07:13:31,091 DEBUG [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-19T07:13:31,092 INFO [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756-splitting/db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta 2024-11-19T07:13:31,098 INFO [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756-splitting/db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta after 3ms 2024-11-19T07:13:31,106 INFO [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(310): Open hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756-splitting/db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta took 17ms 2024-11-19T07:13:31,115 DEBUG [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-19T07:13:31,122 INFO [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta.temp 2024-11-19T07:13:31,125 INFO [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta.temp 2024-11-19T07:13:31,126 DEBUG [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756-splitting/db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta so closing down 2024-11-19T07:13:31,126 DEBUG [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-19T07:13:31,131 INFO [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(145): 3 split writer threads finished 2024-11-19T07:13:31,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741864_1040 (size=33508) 2024-11-19T07:13:31,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741864_1040 (size=33508) 2024-11-19T07:13:31,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741864_1040 (size=33508) 2024-11-19T07:13:31,142 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta.temp (wrote 61 edits, skipped 0 edits in 4 ms) 2024-11-19T07:13:31,144 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta.temp to hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-19T07:13:31,147 INFO [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(425): Processed 62 edits across 1 Regions in 37 ms; skipped=1; WAL=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756-splitting/db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false 2024-11-19T07:13:31,147 DEBUG [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(428): Completed split of hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756-splitting/db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, journal: Splitting hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756-splitting/db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, size=33.0 K (33749bytes) at 1732000411091Creating recovered edits writer path=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta.temp at 1732000411125 (+34 ms)Finishing writing output for hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756-splitting/db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta so closing down at 1732000411126 (+1 ms)3 split writer threads finished at 1732000411131 (+5 ms)Closed recovered edits writer path=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta.temp (wrote 61 edits, skipped 0 edits in 4 ms) at 1732000411142 (+11 ms)Rename recovered edits hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta.temp to hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 at 1732000411144 (+2 ms)Processed 62 edits across 1 Regions in 37 ms; skipped=1; WAL=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756-splitting/db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false at 1732000411147 (+3 ms) 2024-11-19T07:13:31,148 DEBUG [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756-splitting/db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta 2024-11-19T07:13:31,149 DEBUG [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-11-19T07:13:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-11-19T07:13:31,158 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756-splitting/db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta to hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/oldWALs 2024-11-19T07:13:31,163 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-19T07:13:31,163 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 in 262 msec 2024-11-19T07:13:31,164 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:31,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-19T07:13:31,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 in 278 msec 2024-11-19T07:13:31,169 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756-splitting dir is empty, no logs to split. 2024-11-19T07:13:31,170 INFO [PEWorker-1 {}] master.SplitWALManager(105): db0afee3eab9,37853,1732000404756 WAL count=0, meta=true 2024-11-19T07:13:31,170 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if db0afee3eab9,37853,1732000404756 WAL splitting is done? wals=0, meta=true 2024-11-19T07:13:31,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-19T07:13:31,173 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T07:13:31,174 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-19T07:13:31,303 INFO [regionserver/db0afee3eab9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-19T07:13:31,303 INFO [regionserver/db0afee3eab9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-19T07:13:31,324 DEBUG [db0afee3eab9:43779 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-19T07:13:31,325 DEBUG [db0afee3eab9:43779 {}] balancer.BalancerClusterState(204): Hosts are {db0afee3eab9=0} racks are {/default-rack=0} 2024-11-19T07:13:31,325 DEBUG [db0afee3eab9:43779 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-19T07:13:31,325 DEBUG [db0afee3eab9:43779 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-19T07:13:31,325 DEBUG [db0afee3eab9:43779 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-19T07:13:31,325 DEBUG [db0afee3eab9:43779 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-19T07:13:31,325 INFO [db0afee3eab9:43779 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-19T07:13:31,325 INFO [db0afee3eab9:43779 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-19T07:13:31,325 DEBUG [db0afee3eab9:43779 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-19T07:13:31,326 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:31,327 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,42435,1732000404708, state=OPENING 2024-11-19T07:13:31,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:31,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:31,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:31,343 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T07:13:31,343 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:31,343 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:31,343 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:31,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708}] 2024-11-19T07:13:31,503 INFO [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-19T07:13:31,503 INFO [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T07:13:31,504 INFO [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-19T07:13:31,507 INFO [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db0afee3eab9%2C42435%2C1732000404708.meta, suffix=.meta, logDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,42435,1732000404708, archiveDir=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/oldWALs, maxLogs=32 2024-11-19T07:13:31,525 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,42435,1732000404708/db0afee3eab9%2C42435%2C1732000404708.meta.1732000411511.meta, exclude list is [], retry=0 2024-11-19T07:13:31,528 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39225,DS-78acb0be-8b81-4317-958b-6530bbb93025,DISK] 2024-11-19T07:13:31,528 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44773,DS-671bf5ab-8128-4e2b-addb-a3ea7fdb8afc,DISK] 2024-11-19T07:13:31,528 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41661,DS-e0056ec2-0746-4cc1-bce1-20ef1e1aeac2,DISK] 2024-11-19T07:13:31,530 INFO [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,42435,1732000404708/db0afee3eab9%2C42435%2C1732000404708.meta.1732000411511.meta 2024-11-19T07:13:31,530 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32827:32827),(127.0.0.1/127.0.0.1:33965:33965),(127.0.0.1/127.0.0.1:39091:39091)] 2024-11-19T07:13:31,530 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-19T07:13:31,531 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-19T07:13:31,531 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-19T07:13:31,531 INFO [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-19T07:13:31,531 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-19T07:13:31,531 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:31,531 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-19T07:13:31,532 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-19T07:13:31,534 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T07:13:31,535 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T07:13:31,535 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:31,535 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T07:13:31,535 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T07:13:31,536 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T07:13:31,536 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:31,537 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T07:13:31,537 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T07:13:31,538 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T07:13:31,538 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:31,539 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T07:13:31,539 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T07:13:31,540 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T07:13:31,540 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:31,540 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T07:13:31,541 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T07:13:31,541 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740 2024-11-19T07:13:31,543 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740 2024-11-19T07:13:31,544 INFO [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-19T07:13:31,558 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5793): Applied 277, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=64, path=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-19T07:13:31,561 INFO [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=29.41 KB heapSize=50.67 KB 2024-11-19T07:13:31,629 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/.tmp/info/2afe06835f314da7b5442a6b21d72810 is 138, key is test,bbb,1732000407304.12dcc969f79695b5f0d224b605268d2d./info:regioninfo/1732000408260/Put/seqid=0 2024-11-19T07:13:31,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741866_1042 (size=36372) 2024-11-19T07:13:31,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741866_1042 (size=36372) 2024-11-19T07:13:31,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741866_1042 (size=36372) 2024-11-19T07:13:31,643 INFO [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.26 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/.tmp/info/2afe06835f314da7b5442a6b21d72810 2024-11-19T07:13:31,707 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/.tmp/ns/37258f66c192456196694eb31632c359 is 43, key is default/ns:d/1732000407071/Put/seqid=0 2024-11-19T07:13:31,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741867_1043 (size=5153) 2024-11-19T07:13:31,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741867_1043 (size=5153) 2024-11-19T07:13:31,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741867_1043 (size=5153) 2024-11-19T07:13:31,721 INFO [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/.tmp/ns/37258f66c192456196694eb31632c359 2024-11-19T07:13:31,746 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/.tmp/table/84e809c0fb4241e8b30973d01ea70442 is 40, key is test/table:state/1732000408485/Put/seqid=0 2024-11-19T07:13:31,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741868_1044 (size=5165) 2024-11-19T07:13:31,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741868_1044 (size=5165) 2024-11-19T07:13:31,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741868_1044 (size=5165) 2024-11-19T07:13:31,756 INFO [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/.tmp/table/84e809c0fb4241e8b30973d01ea70442 2024-11-19T07:13:31,766 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/.tmp/info/2afe06835f314da7b5442a6b21d72810 as hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/info/2afe06835f314da7b5442a6b21d72810 2024-11-19T07:13:31,776 INFO [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/info/2afe06835f314da7b5442a6b21d72810, entries=273, sequenceid=64, filesize=35.5 K 2024-11-19T07:13:31,778 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/.tmp/ns/37258f66c192456196694eb31632c359 as hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/ns/37258f66c192456196694eb31632c359 2024-11-19T07:13:31,786 INFO [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/ns/37258f66c192456196694eb31632c359, entries=2, sequenceid=64, filesize=5.0 K 2024-11-19T07:13:31,787 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/.tmp/table/84e809c0fb4241e8b30973d01ea70442 as hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/table/84e809c0fb4241e8b30973d01ea70442 2024-11-19T07:13:31,796 INFO [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/table/84e809c0fb4241e8b30973d01ea70442, entries=2, sequenceid=64, filesize=5.0 K 2024-11-19T07:13:31,800 INFO [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(3140): Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 237ms, sequenceid=64, compaction requested=false; wal=null 2024-11-19T07:13:31,801 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-19T07:13:31,804 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-19T07:13:31,806 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T07:13:31,806 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T07:13:31,806 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T07:13:31,808 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T07:13:31,810 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/recovered.edits/64.seqid, newMaxSeqId=64, maxSeqId=1 2024-11-19T07:13:31,811 INFO [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=65; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59977370, jitterRate=-0.1062675416469574}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T07:13:31,812 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-19T07:13:31,812 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732000411532Writing region info on filesystem at 1732000411532Initializing all the Stores at 1732000411533 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732000411533Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732000411533Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000411534 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732000411534Obtaining lock to block concurrent updates at 1732000411561 (+27 ms)Preparing flush snapshotting stores in 1588230740 at 1732000411561Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=30111, getHeapSize=51824, getOffHeapSize=0, getCellsCount=277 at 1732000411564 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732000411564Flushing 1588230740/info: creating writer at 1732000411566 (+2 ms)Flushing 1588230740/info: appending metadata at 1732000411618 (+52 ms)Flushing 1588230740/info: closing flushed file at 1732000411621 (+3 ms)Flushing 1588230740/ns: creating writer at 1732000411690 (+69 ms)Flushing 1588230740/ns: appending metadata at 1732000411706 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732000411706Flushing 1588230740/table: creating writer at 1732000411730 (+24 ms)Flushing 1588230740/table: appending metadata at 1732000411745 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732000411745Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c70eaf5: reopening flushed file at 1732000411765 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b147833: reopening flushed file at 1732000411777 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ae1c6ec: reopening flushed file at 1732000411786 (+9 ms)Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 237ms, sequenceid=64, compaction requested=false; wal=null at 1732000411800 (+14 ms)Cleaning up temporary data from old regions at 1732000411806 (+6 ms)Running coprocessor post-open hooks at 1732000411812 (+6 ms)Region opened successfully at 1732000411812 2024-11-19T07:13:31,813 INFO [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=64, masterSystemTime=1732000411496 2024-11-19T07:13:31,816 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-19T07:13:31,816 INFO [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-19T07:13:31,817 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=65, regionLocation=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:31,818 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,42435,1732000404708, state=OPEN 2024-11-19T07:13:31,831 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=db0afee3eab9,42435,1732000404708, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T07:13:31,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:31,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:31,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:31,838 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=64, ppid=63, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:31,838 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:31,838 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:31,838 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:31,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-19T07:13:31,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 in 495 msec 2024-11-19T07:13:31,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=60 2024-11-19T07:13:31,846 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 672 msec 2024-11-19T07:13:31,846 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): db0afee3eab9,37853,1732000404756 had 9 regions 2024-11-19T07:13:31,847 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true, isMeta: false 2024-11-19T07:13:31,850 INFO [PEWorker-1 {}] master.SplitWALManager(105): db0afee3eab9,37853,1732000404756 WAL count=1, meta=false 2024-11-19T07:13:31,850 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457}] 2024-11-19T07:13:31,852 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:31,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708}] 2024-11-19T07:13:32,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42435 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=66 2024-11-19T07:13:32,030 INFO [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(299): Splitting hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756-splitting/db0afee3eab9%2C37853%2C1732000404756.1732000406457, size=2.7 K (2774bytes) 2024-11-19T07:13:32,030 INFO [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756-splitting/db0afee3eab9%2C37853%2C1732000404756.1732000406457 2024-11-19T07:13:32,031 INFO [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756-splitting/db0afee3eab9%2C37853%2C1732000404756.1732000406457 after 1ms 2024-11-19T07:13:32,034 INFO [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(310): Open hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756-splitting/db0afee3eab9%2C37853%2C1732000404756.1732000406457 took 4ms 2024-11-19T07:13:32,037 DEBUG [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 08f6487d04261d6f0713b4b5adeacf2e: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-19T07:13:32,039 DEBUG [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for e880c03f4c433baa6242dd65ef2d8486: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-19T07:13:32,041 DEBUG [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 7d9d31e1a2abc752a2056ed0e3a3fcec: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-19T07:13:32,043 DEBUG [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 2f38687f659a16d9e2a781f2a75d514a: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-19T07:13:32,046 DEBUG [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 66fe58970f9c4a1cba9c3e3da843ed3d: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-19T07:13:32,048 DEBUG [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for bbb518f4693490310eca052bf28f87fc: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-19T07:13:32,050 DEBUG [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 3aa6780c71c61c778946d2086357143d: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-19T07:13:32,052 DEBUG [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 2df93dc30d9c55d765ada1f50e320789: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-19T07:13:32,055 DEBUG [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 74657be6232ad22aaca548d82491de41: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-19T07:13:32,055 DEBUG [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756-splitting/db0afee3eab9%2C37853%2C1732000404756.1732000406457 so closing down 2024-11-19T07:13:32,055 DEBUG [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-19T07:13:32,055 INFO [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(145): 3 split writer threads finished 2024-11-19T07:13:32,055 INFO [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(425): Processed 10 edits across 0 Regions in 21 ms; skipped=10; WAL=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756-splitting/db0afee3eab9%2C37853%2C1732000404756.1732000406457, size=2.7 K, length=2774, corrupted=false, cancelled=false 2024-11-19T07:13:32,055 DEBUG [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(428): Completed split of hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756-splitting/db0afee3eab9%2C37853%2C1732000404756.1732000406457, journal: Splitting hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756-splitting/db0afee3eab9%2C37853%2C1732000404756.1732000406457, size=2.7 K (2774bytes) at 1732000412030Finishing writing output for hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756-splitting/db0afee3eab9%2C37853%2C1732000404756.1732000406457 so closing down at 1732000412055 (+25 ms)3 split writer threads finished at 1732000412055Processed 10 edits across 0 Regions in 21 ms; skipped=10; WAL=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756-splitting/db0afee3eab9%2C37853%2C1732000404756.1732000406457, size=2.7 K, length=2774, corrupted=false, cancelled=false at 1732000412055 2024-11-19T07:13:32,055 DEBUG [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756-splitting/db0afee3eab9%2C37853%2C1732000404756.1732000406457 2024-11-19T07:13:32,056 DEBUG [RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0-0 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-11-19T07:13:32,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43779 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-11-19T07:13:32,061 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756-splitting/db0afee3eab9%2C37853%2C1732000404756.1732000406457 to hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/oldWALs 2024-11-19T07:13:32,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-19T07:13:32,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 in 208 msec 2024-11-19T07:13:32,065 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=60 2024-11-19T07:13:32,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 in 215 msec 2024-11-19T07:13:32,070 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/WALs/db0afee3eab9,37853,1732000404756-splitting dir is empty, no logs to split. 2024-11-19T07:13:32,070 INFO [PEWorker-1 {}] master.SplitWALManager(105): db0afee3eab9,37853,1732000404756 WAL count=0, meta=false 2024-11-19T07:13:32,070 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if db0afee3eab9,37853,1732000404756 WAL splitting is done? wals=0, meta=false 2024-11-19T07:13:32,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN}, {pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN}, {pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN}, {pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN}, {pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN}, {pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN}, {pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN}, {pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN}] 2024-11-19T07:13:32,075 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN 2024-11-19T07:13:32,075 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-19T07:13:32,076 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN 2024-11-19T07:13:32,077 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-19T07:13:32,078 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN 2024-11-19T07:13:32,079 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-19T07:13:32,080 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN 2024-11-19T07:13:32,081 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-19T07:13:32,082 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN 2024-11-19T07:13:32,082 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-19T07:13:32,083 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:32,084 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-19T07:13:32,085 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:32,086 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-19T07:13:32,087 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:32,089 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-19T07:13:32,226 DEBUG [db0afee3eab9:43779 {}] balancer.BalancerClusterState(204): Hosts are {db0afee3eab9=0} racks are {/default-rack=0} 2024-11-19T07:13:32,227 DEBUG [db0afee3eab9:43779 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-19T07:13:32,227 DEBUG [db0afee3eab9:43779 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-19T07:13:32,227 DEBUG [db0afee3eab9:43779 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-19T07:13:32,227 DEBUG [db0afee3eab9:43779 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-19T07:13:32,227 INFO [db0afee3eab9:43779 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-19T07:13:32,227 INFO [db0afee3eab9:43779 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-19T07:13:32,227 DEBUG [db0afee3eab9:43779 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-19T07:13:32,228 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=08f6487d04261d6f0713b4b5adeacf2e, regionState=OPENING, regionLocation=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,229 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=2df93dc30d9c55d765ada1f50e320789, regionState=OPENING, regionLocation=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,230 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=66fe58970f9c4a1cba9c3e3da843ed3d, regionState=OPENING, regionLocation=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,230 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server db0afee3eab9:37853 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: db0afee3eab9/172.17.0.2:37853 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T07:13:32,231 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=7d9d31e1a2abc752a2056ed0e3a3fcec, regionState=OPENING, regionLocation=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,232 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=3aa6780c71c61c778946d2086357143d, regionState=OPENING, regionLocation=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,233 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,37853,1732000404756, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,37853,1732000404756, seqNum=-1, error=java.net.ConnectException: Call to address=db0afee3eab9:37853 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: db0afee3eab9/172.17.0.2:37853 2024-11-19T07:13:32,233 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=2f38687f659a16d9e2a781f2a75d514a, regionState=OPENING, regionLocation=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,234 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=bbb518f4693490310eca052bf28f87fc, regionState=OPENING, regionLocation=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,234 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,37853,1732000404756, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-19T07:13:32,234 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,37853,1732000404756, seqNum=-1 from cache 2024-11-19T07:13:32,235 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=e880c03f4c433baa6242dd65ef2d8486, regionState=OPENING, regionLocation=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,235 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T07:13:32,235 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,42435,1732000404708, seqNum=-1] 2024-11-19T07:13:32,236 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address db0afee3eab9:37853 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: db0afee3eab9/172.17.0.2:37853 2024-11-19T07:13:32,236 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T07:13:32,237 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server db0afee3eab9:37853 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: db0afee3eab9/172.17.0.2:37853 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T07:13:32,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,37853,1732000404756, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,42435,1732000404708, seqNum=-1, error=java.net.ConnectException: Call to address=db0afee3eab9:37853 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: db0afee3eab9/172.17.0.2:37853 2024-11-19T07:13:32,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,37853,1732000404756, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,42435,1732000404708, seqNum=-1, error=java.net.ConnectException: Call to address=db0afee3eab9:37853 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: db0afee3eab9/172.17.0.2:37853 2024-11-19T07:13:32,238 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45863, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T07:13:32,239 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,37853,1732000404756, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,42435,1732000404708, seqNum=-1, error=java.net.ConnectException: Call to address=db0afee3eab9:37853 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: db0afee3eab9/172.17.0.2:37853 2024-11-19T07:13:32,239 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,37853,1732000404756, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,42435,1732000404708, seqNum=-1, error=java.net.ConnectException: Call to address=db0afee3eab9:37853 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: db0afee3eab9/172.17.0.2:37853 2024-11-19T07:13:32,239 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,37853,1732000404756, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,42435,1732000404708, seqNum=-1, error=java.net.ConnectException: Call to address=db0afee3eab9:37853 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: db0afee3eab9/172.17.0.2:37853 2024-11-19T07:13:32,239 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,37853,1732000404756, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,42435,1732000404708, seqNum=-1, error=java.net.ConnectException: Call to address=db0afee3eab9:37853 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: db0afee3eab9/172.17.0.2:37853 2024-11-19T07:13:32,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address db0afee3eab9:37853 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: db0afee3eab9/172.17.0.2:37853 2024-11-19T07:13:32,241 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN because future has completed 2024-11-19T07:13:32,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure e880c03f4c433baa6242dd65ef2d8486, server=db0afee3eab9,34425,1732000404604}] 2024-11-19T07:13:32,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN because future has completed 2024-11-19T07:13:32,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08f6487d04261d6f0713b4b5adeacf2e, server=db0afee3eab9,42435,1732000404708}] 2024-11-19T07:13:32,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN because future has completed 2024-11-19T07:13:32,355 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN because future has completed 2024-11-19T07:13:32,357 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN because future has completed 2024-11-19T07:13:32,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66fe58970f9c4a1cba9c3e3da843ed3d, server=db0afee3eab9,42435,1732000404708}] 2024-11-19T07:13:32,359 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN because future has completed 2024-11-19T07:13:32,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN because future has completed 2024-11-19T07:13:32,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3aa6780c71c61c778946d2086357143d, server=db0afee3eab9,34425,1732000404604}] 2024-11-19T07:13:32,366 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN because future has completed 2024-11-19T07:13:32,374 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604}] 2024-11-19T07:13:32,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7d9d31e1a2abc752a2056ed0e3a3fcec, server=db0afee3eab9,42435,1732000404708}] 2024-11-19T07:13:32,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2df93dc30d9c55d765ada1f50e320789, server=db0afee3eab9,42435,1732000404708}] 2024-11-19T07:13:32,395 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbb518f4693490310eca052bf28f87fc, server=db0afee3eab9,34425,1732000404604}] 2024-11-19T07:13:32,399 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,ttt,1732000407304.2f38687f659a16d9e2a781f2a75d514a. 2024-11-19T07:13:32,399 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 2f38687f659a16d9e2a781f2a75d514a, NAME => 'test,ttt,1732000407304.2f38687f659a16d9e2a781f2a75d514a.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-19T07:13:32,399 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2f38687f659a16d9e2a781f2a75d514a 2024-11-19T07:13:32,399 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,ttt,1732000407304.2f38687f659a16d9e2a781f2a75d514a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:32,400 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 2f38687f659a16d9e2a781f2a75d514a 2024-11-19T07:13:32,400 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 2f38687f659a16d9e2a781f2a75d514a 2024-11-19T07:13:32,418 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T07:13:32,425 INFO [StoreOpener-2f38687f659a16d9e2a781f2a75d514a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2f38687f659a16d9e2a781f2a75d514a 2024-11-19T07:13:32,428 INFO [StoreOpener-2f38687f659a16d9e2a781f2a75d514a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2f38687f659a16d9e2a781f2a75d514a columnFamilyName family 2024-11-19T07:13:32,428 DEBUG [StoreOpener-2f38687f659a16d9e2a781f2a75d514a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:32,429 INFO [StoreOpener-2f38687f659a16d9e2a781f2a75d514a-1 {}] regionserver.HStore(327): Store=2f38687f659a16d9e2a781f2a75d514a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:32,437 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 2f38687f659a16d9e2a781f2a75d514a 2024-11-19T07:13:32,438 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/2f38687f659a16d9e2a781f2a75d514a 2024-11-19T07:13:32,440 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/2f38687f659a16d9e2a781f2a75d514a 2024-11-19T07:13:32,444 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 2f38687f659a16d9e2a781f2a75d514a 2024-11-19T07:13:32,444 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 2f38687f659a16d9e2a781f2a75d514a 2024-11-19T07:13:32,448 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 2f38687f659a16d9e2a781f2a75d514a 2024-11-19T07:13:32,449 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 2f38687f659a16d9e2a781f2a75d514a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70407631, jitterRate=0.04915545880794525}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:32,450 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2f38687f659a16d9e2a781f2a75d514a 2024-11-19T07:13:32,450 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 2f38687f659a16d9e2a781f2a75d514a: Running coprocessor pre-open hook at 1732000412400Writing region info on filesystem at 1732000412400Initializing all the Stores at 1732000412401 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000412402 (+1 ms)Cleaning up temporary data from old regions at 1732000412444 (+42 ms)Running coprocessor post-open hooks at 1732000412450 (+6 ms)Region opened successfully at 1732000412450 2024-11-19T07:13:32,451 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1732000407304.2f38687f659a16d9e2a781f2a75d514a., pid=79, masterSystemTime=1732000412393 2024-11-19T07:13:32,456 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1732000407304.2f38687f659a16d9e2a781f2a75d514a. 2024-11-19T07:13:32,456 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,ttt,1732000407304.2f38687f659a16d9e2a781f2a75d514a. 2024-11-19T07:13:32,456 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,eee,1732000407304.e880c03f4c433baa6242dd65ef2d8486. 2024-11-19T07:13:32,456 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => e880c03f4c433baa6242dd65ef2d8486, NAME => 'test,eee,1732000407304.e880c03f4c433baa6242dd65ef2d8486.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-19T07:13:32,457 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e880c03f4c433baa6242dd65ef2d8486 2024-11-19T07:13:32,457 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,eee,1732000407304.e880c03f4c433baa6242dd65ef2d8486.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:32,457 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for e880c03f4c433baa6242dd65ef2d8486 2024-11-19T07:13:32,457 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for e880c03f4c433baa6242dd65ef2d8486 2024-11-19T07:13:32,458 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-19T07:13:32,458 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,459 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:32,476 INFO [StoreOpener-e880c03f4c433baa6242dd65ef2d8486-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e880c03f4c433baa6242dd65ef2d8486 2024-11-19T07:13:32,478 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-11-19T07:13:32,485 INFO [StoreOpener-e880c03f4c433baa6242dd65ef2d8486-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e880c03f4c433baa6242dd65ef2d8486 columnFamilyName family 2024-11-19T07:13:32,485 DEBUG [StoreOpener-e880c03f4c433baa6242dd65ef2d8486-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:32,485 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T07:13:32,485 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=82, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbb518f4693490310eca052bf28f87fc, server=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:32,486 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=82, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbb518f4693490310eca052bf28f87fc, server=db0afee3eab9,34425,1732000404604 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:32,486 INFO [StoreOpener-e880c03f4c433baa6242dd65ef2d8486-1 {}] regionserver.HStore(327): Store=e880c03f4c433baa6242dd65ef2d8486/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:32,487 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for e880c03f4c433baa6242dd65ef2d8486 2024-11-19T07:13:32,488 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/e880c03f4c433baa6242dd65ef2d8486 2024-11-19T07:13:32,492 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/e880c03f4c433baa6242dd65ef2d8486 2024-11-19T07:13:32,494 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for e880c03f4c433baa6242dd65ef2d8486 2024-11-19T07:13:32,494 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for e880c03f4c433baa6242dd65ef2d8486 2024-11-19T07:13:32,497 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for e880c03f4c433baa6242dd65ef2d8486 2024-11-19T07:13:32,499 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened e880c03f4c433baa6242dd65ef2d8486; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67087434, jitterRate=-3.193318843841553E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:32,499 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e880c03f4c433baa6242dd65ef2d8486 2024-11-19T07:13:32,500 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for e880c03f4c433baa6242dd65ef2d8486: Running coprocessor pre-open hook at 1732000412457Writing region info on filesystem at 1732000412457Initializing all the Stores at 1732000412462 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000412462Cleaning up temporary data from old regions at 1732000412494 (+32 ms)Running coprocessor post-open hooks at 1732000412499 (+5 ms)Region opened successfully at 1732000412499 2024-11-19T07:13:32,504 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732000407304.e880c03f4c433baa6242dd65ef2d8486., pid=75, masterSystemTime=1732000412393 2024-11-19T07:13:32,508 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732000407304.e880c03f4c433baa6242dd65ef2d8486. 2024-11-19T07:13:32,509 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,eee,1732000407304.e880c03f4c433baa6242dd65ef2d8486. 2024-11-19T07:13:32,509 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,yyy,1732000407304.3aa6780c71c61c778946d2086357143d. 2024-11-19T07:13:32,510 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open test,ccc,1732000407304.7d9d31e1a2abc752a2056ed0e3a3fcec. 2024-11-19T07:13:32,511 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => 7d9d31e1a2abc752a2056ed0e3a3fcec, NAME => 'test,ccc,1732000407304.7d9d31e1a2abc752a2056ed0e3a3fcec.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-19T07:13:32,511 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 3aa6780c71c61c778946d2086357143d, NAME => 'test,yyy,1732000407304.3aa6780c71c61c778946d2086357143d.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-19T07:13:32,511 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7d9d31e1a2abc752a2056ed0e3a3fcec 2024-11-19T07:13:32,511 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated test,ccc,1732000407304.7d9d31e1a2abc752a2056ed0e3a3fcec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:32,511 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3aa6780c71c61c778946d2086357143d 2024-11-19T07:13:32,511 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,yyy,1732000407304.3aa6780c71c61c778946d2086357143d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:32,511 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for 7d9d31e1a2abc752a2056ed0e3a3fcec 2024-11-19T07:13:32,511 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for 7d9d31e1a2abc752a2056ed0e3a3fcec 2024-11-19T07:13:32,511 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 3aa6780c71c61c778946d2086357143d 2024-11-19T07:13:32,511 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 3aa6780c71c61c778946d2086357143d 2024-11-19T07:13:32,524 INFO [StoreOpener-7d9d31e1a2abc752a2056ed0e3a3fcec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7d9d31e1a2abc752a2056ed0e3a3fcec 2024-11-19T07:13:32,525 INFO [StoreOpener-3aa6780c71c61c778946d2086357143d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3aa6780c71c61c778946d2086357143d 2024-11-19T07:13:32,526 INFO [StoreOpener-7d9d31e1a2abc752a2056ed0e3a3fcec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7d9d31e1a2abc752a2056ed0e3a3fcec columnFamilyName family 2024-11-19T07:13:32,526 DEBUG [StoreOpener-7d9d31e1a2abc752a2056ed0e3a3fcec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:32,527 INFO [StoreOpener-7d9d31e1a2abc752a2056ed0e3a3fcec-1 {}] regionserver.HStore(327): Store=7d9d31e1a2abc752a2056ed0e3a3fcec/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:32,527 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for 7d9d31e1a2abc752a2056ed0e3a3fcec 2024-11-19T07:13:32,528 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/7d9d31e1a2abc752a2056ed0e3a3fcec 2024-11-19T07:13:32,531 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/7d9d31e1a2abc752a2056ed0e3a3fcec 2024-11-19T07:13:32,531 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for 7d9d31e1a2abc752a2056ed0e3a3fcec 2024-11-19T07:13:32,531 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for 7d9d31e1a2abc752a2056ed0e3a3fcec 2024-11-19T07:13:32,532 INFO [StoreOpener-3aa6780c71c61c778946d2086357143d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3aa6780c71c61c778946d2086357143d columnFamilyName family 2024-11-19T07:13:32,532 DEBUG [StoreOpener-3aa6780c71c61c778946d2086357143d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:32,533 INFO [StoreOpener-3aa6780c71c61c778946d2086357143d-1 {}] regionserver.HStore(327): Store=3aa6780c71c61c778946d2086357143d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:32,533 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 3aa6780c71c61c778946d2086357143d 2024-11-19T07:13:32,534 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/3aa6780c71c61c778946d2086357143d 2024-11-19T07:13:32,535 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for 7d9d31e1a2abc752a2056ed0e3a3fcec 2024-11-19T07:13:32,536 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/3aa6780c71c61c778946d2086357143d 2024-11-19T07:13:32,537 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened 7d9d31e1a2abc752a2056ed0e3a3fcec; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71933861, jitterRate=0.07189805805683136}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:32,537 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7d9d31e1a2abc752a2056ed0e3a3fcec 2024-11-19T07:13:32,537 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 3aa6780c71c61c778946d2086357143d 2024-11-19T07:13:32,537 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 3aa6780c71c61c778946d2086357143d 2024-11-19T07:13:32,537 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for 7d9d31e1a2abc752a2056ed0e3a3fcec: Running coprocessor pre-open hook at 1732000412512Writing region info on filesystem at 1732000412512Initializing all the Stores at 1732000412513 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000412513Cleaning up temporary data from old regions at 1732000412531 (+18 ms)Running coprocessor post-open hooks at 1732000412537 (+6 ms)Region opened successfully at 1732000412537 2024-11-19T07:13:32,541 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732000407304.7d9d31e1a2abc752a2056ed0e3a3fcec., pid=80, masterSystemTime=1732000412504 2024-11-19T07:13:32,545 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 3aa6780c71c61c778946d2086357143d 2024-11-19T07:13:32,549 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1732000407304.7d9d31e1a2abc752a2056ed0e3a3fcec. 2024-11-19T07:13:32,549 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened test,ccc,1732000407304.7d9d31e1a2abc752a2056ed0e3a3fcec. 2024-11-19T07:13:32,549 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,vvv,1732000407304.08f6487d04261d6f0713b4b5adeacf2e. 2024-11-19T07:13:32,549 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 08f6487d04261d6f0713b4b5adeacf2e, NAME => 'test,vvv,1732000407304.08f6487d04261d6f0713b4b5adeacf2e.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-19T07:13:32,550 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 08f6487d04261d6f0713b4b5adeacf2e 2024-11-19T07:13:32,550 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,vvv,1732000407304.08f6487d04261d6f0713b4b5adeacf2e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:32,550 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 08f6487d04261d6f0713b4b5adeacf2e 2024-11-19T07:13:32,550 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 08f6487d04261d6f0713b4b5adeacf2e 2024-11-19T07:13:32,552 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 3aa6780c71c61c778946d2086357143d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61268009, jitterRate=-0.08703552186489105}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:32,553 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3aa6780c71c61c778946d2086357143d 2024-11-19T07:13:32,553 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 3aa6780c71c61c778946d2086357143d: Running coprocessor pre-open hook at 1732000412512Writing region info on filesystem at 1732000412512Initializing all the Stores at 1732000412513 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000412513Cleaning up temporary data from old regions at 1732000412537 (+24 ms)Running coprocessor post-open hooks at 1732000412553 (+16 ms)Region opened successfully at 1732000412553 2024-11-19T07:13:32,560 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1732000407304.3aa6780c71c61c778946d2086357143d., pid=78, masterSystemTime=1732000412393 2024-11-19T07:13:32,563 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1732000407304.3aa6780c71c61c778946d2086357143d. 2024-11-19T07:13:32,563 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,yyy,1732000407304.3aa6780c71c61c778946d2086357143d. 2024-11-19T07:13:32,564 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(132): Open test,kkk,1732000407304.bbb518f4693490310eca052bf28f87fc. 2024-11-19T07:13:32,564 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7752): Opening region: {ENCODED => bbb518f4693490310eca052bf28f87fc, NAME => 'test,kkk,1732000407304.bbb518f4693490310eca052bf28f87fc.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-19T07:13:32,564 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bbb518f4693490310eca052bf28f87fc 2024-11-19T07:13:32,564 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(898): Instantiated test,kkk,1732000407304.bbb518f4693490310eca052bf28f87fc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:32,565 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7794): checking encryption for bbb518f4693490310eca052bf28f87fc 2024-11-19T07:13:32,565 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7797): checking classloading for bbb518f4693490310eca052bf28f87fc 2024-11-19T07:13:32,565 INFO [StoreOpener-08f6487d04261d6f0713b4b5adeacf2e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 08f6487d04261d6f0713b4b5adeacf2e 2024-11-19T07:13:32,567 INFO [StoreOpener-08f6487d04261d6f0713b4b5adeacf2e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08f6487d04261d6f0713b4b5adeacf2e columnFamilyName family 2024-11-19T07:13:32,567 DEBUG [StoreOpener-08f6487d04261d6f0713b4b5adeacf2e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:32,568 INFO [StoreOpener-08f6487d04261d6f0713b4b5adeacf2e-1 {}] regionserver.HStore(327): Store=08f6487d04261d6f0713b4b5adeacf2e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:32,568 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 08f6487d04261d6f0713b4b5adeacf2e 2024-11-19T07:13:32,568 INFO [StoreOpener-bbb518f4693490310eca052bf28f87fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bbb518f4693490310eca052bf28f87fc 2024-11-19T07:13:32,569 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/08f6487d04261d6f0713b4b5adeacf2e 2024-11-19T07:13:32,570 INFO [StoreOpener-bbb518f4693490310eca052bf28f87fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbb518f4693490310eca052bf28f87fc columnFamilyName family 2024-11-19T07:13:32,570 DEBUG [StoreOpener-bbb518f4693490310eca052bf28f87fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:32,571 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/08f6487d04261d6f0713b4b5adeacf2e 2024-11-19T07:13:32,571 INFO [StoreOpener-bbb518f4693490310eca052bf28f87fc-1 {}] regionserver.HStore(327): Store=bbb518f4693490310eca052bf28f87fc/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:32,571 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1038): replaying wal for bbb518f4693490310eca052bf28f87fc 2024-11-19T07:13:32,572 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 08f6487d04261d6f0713b4b5adeacf2e 2024-11-19T07:13:32,572 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 08f6487d04261d6f0713b4b5adeacf2e 2024-11-19T07:13:32,572 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/bbb518f4693490310eca052bf28f87fc 2024-11-19T07:13:32,574 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/bbb518f4693490310eca052bf28f87fc 2024-11-19T07:13:32,575 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1048): stopping wal replay for bbb518f4693490310eca052bf28f87fc 2024-11-19T07:13:32,575 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 08f6487d04261d6f0713b4b5adeacf2e 2024-11-19T07:13:32,575 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1060): Cleaning up temporary data for bbb518f4693490310eca052bf28f87fc 2024-11-19T07:13:32,576 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 08f6487d04261d6f0713b4b5adeacf2e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63993401, jitterRate=-0.0464240163564682}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:32,576 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 08f6487d04261d6f0713b4b5adeacf2e 2024-11-19T07:13:32,576 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 08f6487d04261d6f0713b4b5adeacf2e: Running coprocessor pre-open hook at 1732000412550Writing region info on filesystem at 1732000412550Initializing all the Stores at 1732000412556 (+6 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000412557 (+1 ms)Cleaning up temporary data from old regions at 1732000412572 (+15 ms)Running coprocessor post-open hooks at 1732000412576 (+4 ms)Region opened successfully at 1732000412576 2024-11-19T07:13:32,578 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T07:13:32,578 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732000407304.08f6487d04261d6f0713b4b5adeacf2e., pid=76, masterSystemTime=1732000412504 2024-11-19T07:13:32,578 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:32,578 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T07:13:32,578 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1093): writing seq id for bbb518f4693490310eca052bf28f87fc 2024-11-19T07:13:32,578 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T07:13:32,579 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:32,579 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T07:13:32,579 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T07:13:32,580 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T07:13:32,579 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1114): Opened bbb518f4693490310eca052bf28f87fc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64944582, jitterRate=-0.03225031495094299}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:32,580 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bbb518f4693490310eca052bf28f87fc 2024-11-19T07:13:32,580 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1006): Region open journal for bbb518f4693490310eca052bf28f87fc: Running coprocessor pre-open hook at 1732000412565Writing region info on filesystem at 1732000412565Initializing all the Stores at 1732000412567 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000412567Cleaning up temporary data from old regions at 1732000412575 (+8 ms)Running coprocessor post-open hooks at 1732000412580 (+5 ms)Region opened successfully at 1732000412580 2024-11-19T07:13:32,580 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T07:13:32,581 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T07:13:32,581 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-19T07:13:32,582 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1732000407304.bbb518f4693490310eca052bf28f87fc., pid=82, masterSystemTime=1732000412548 2024-11-19T07:13:32,583 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:32,590 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T07:13:32,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:32,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:32,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=82, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:32,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:32,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T07:13:32,597 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T07:13:32,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, REOPEN/MOVE 2024-11-19T07:13:32,605 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true 2024-11-19T07:13:32,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T07:13:32,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:32,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:32,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:32,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN 2024-11-19T07:13:32,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN 2024-11-19T07:13:32,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN 2024-11-19T07:13:32,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN 2024-11-19T07:13:32,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN 2024-11-19T07:13:32,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure e880c03f4c433baa6242dd65ef2d8486, server=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08f6487d04261d6f0713b4b5adeacf2e, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66fe58970f9c4a1cba9c3e3da843ed3d, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3aa6780c71c61c778946d2086357143d, server=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7d9d31e1a2abc752a2056ed0e3a3fcec, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2df93dc30d9c55d765ada1f50e320789, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=82, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbb518f4693490310eca052bf28f87fc, server=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,607 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,607 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,607 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,607 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,607 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,607 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,607 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,607 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,607 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:32,607 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,607 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:32,607 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,607 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:32,608 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,608 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN 2024-11-19T07:13:32,608 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,608 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN 2024-11-19T07:13:32,608 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,608 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN 2024-11-19T07:13:32,608 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,608 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN 2024-11-19T07:13:32,608 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,608 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN 2024-11-19T07:13:32,608 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure e880c03f4c433baa6242dd65ef2d8486, server=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,608 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08f6487d04261d6f0713b4b5adeacf2e, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,608 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66fe58970f9c4a1cba9c3e3da843ed3d, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,608 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3aa6780c71c61c778946d2086357143d, server=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,609 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,609 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7d9d31e1a2abc752a2056ed0e3a3fcec, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,609 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2df93dc30d9c55d765ada1f50e320789, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,609 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=82, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbb518f4693490310eca052bf28f87fc, server=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,610 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 28 msec 2024-11-19T07:13:32,612 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "db0afee3eab9" port: 42435 start_code: 1732000404708 } transition { transition_code: OPENED region_info { region_id: 1732000407304 table_name { namespace: "default" qualifier: "test" } start_key: "vvv" end_key: "www" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 76 initiating_master_active_time: 1732000404850 }; retry (#0) after 1001ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-19T07:13:32,612 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "db0afee3eab9" port: 34425 start_code: 1732000404604 } transition { transition_code: OPENED region_info { region_id: 1732000407304 table_name { namespace: "default" qualifier: "test" } start_key: "kkk" end_key: "lll" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 82 initiating_master_active_time: 1732000404850 }; retry (#0) after 1000ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-19T07:13:32,615 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=65 2024-11-19T07:13:32,615 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=db0afee3eab9,42435,1732000404708, table=hbase:meta, region=1588230740 2024-11-19T07:13:32,616 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,42435,1732000404708, state=OPEN 2024-11-19T07:13:32,655 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:32,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:32,685 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:32,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:32,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:32,685 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:32,685 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:32,687 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN to state=OFFLINE, location=null, table=test, region=08f6487d04261d6f0713b4b5adeacf2e to restore RIT 2024-11-19T07:13:32,687 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN to state=OFFLINE, location=null, table=test, region=3aa6780c71c61c778946d2086357143d to restore RIT 2024-11-19T07:13:32,687 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN to state=OFFLINE, location=null, table=test, region=2f38687f659a16d9e2a781f2a75d514a to restore RIT 2024-11-19T07:13:32,687 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN to state=OFFLINE, location=null, table=test, region=e880c03f4c433baa6242dd65ef2d8486 to restore RIT 2024-11-19T07:13:32,687 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN to state=OFFLINE, location=null, table=test, region=2df93dc30d9c55d765ada1f50e320789 to restore RIT 2024-11-19T07:13:32,687 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN to state=OFFLINE, location=null, table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d to restore RIT 2024-11-19T07:13:32,688 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN to state=OFFLINE, location=null, table=test, region=bbb518f4693490310eca052bf28f87fc to restore RIT 2024-11-19T07:13:32,688 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN to state=OFFLINE, location=null, table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec to restore RIT 2024-11-19T07:13:32,719 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732000442719 2024-11-19T07:13:32,719 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T07:13:32,721 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:32,733 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T07:13:32,735 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T07:13:32,735 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2df93dc30d9c55d765ada1f50e320789, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:32,735 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2df93dc30d9c55d765ada1f50e320789, server=db0afee3eab9,42435,1732000404708 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:32,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02309bab61f6207d73b9f19985e1fdb9, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dcc969f79695b5f0d224b605268d2d, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d9d31e1a2abc752a2056ed0e3a3fcec, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ece759217c055d9a295034ce69dd86d, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,768 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e880c03f4c433baa6242dd65ef2d8486, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,768 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49b850b07eedb76caa94e4c93ac5c832, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,768 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db4ba820395e8b4a2b38d57f8a222e97, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,769 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66fe58970f9c4a1cba9c3e3da843ed3d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,769 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43676ec232db06c9bf4af64db1ede4f5, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,769 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4421c1189c5fdff3a40dcc58728c41ab, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,769 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbb518f4693490310eca052bf28f87fc, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,770 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82941673e9ad03e5a037091750c3f315, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,770 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,770 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74657be6232ad22aaca548d82491de41, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=5 2024-11-19T07:13:32,771 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf73eccbee4f01f5a86b87f17dcdb8b, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,771 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a637105eb1d8b75696b3cfabe5d8173, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,771 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df93dc30d9c55d765ada1f50e320789, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,771 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dba97405d3b35647d42d266df5e81cec, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d703151d82f3a429f348967988283542, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f38687f659a16d9e2a781f2a75d514a, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9357ae6c6e8337f25dacbf0a10701ef6, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08f6487d04261d6f0713b4b5adeacf2e, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=611b07fbb4fb5e3ea7255d9fa0f9fe40, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=126b77a274d2ba63b12dd23c9e7ababb, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3aa6780c71c61c778946d2086357143d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,774 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32d987cca115e6f476cdb5686be2b726, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,774 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T07:13:32,774 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732000472774 2024-11-19T07:13:32,774 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732000532774 2024-11-19T07:13:32,774 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 54 msec 2024-11-19T07:13:32,774 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T07:13:32,774 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:32,774 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T07:13:32,774 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T07:13:32,775 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T07:13:32,775 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T07:13:32,775 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T07:13:32,775 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T07:13:32,777 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-19T07:13:32,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T07:13:32,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:32,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:32,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,785 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,785 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,785 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,785 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:32,785 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:32,785 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,785 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T07:13:32,787 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T07:13:32,788 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, REOPEN/MOVE 2024-11-19T07:13:32,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true 2024-11-19T07:13:32,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T07:13:32,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:32,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:32,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:32,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN 2024-11-19T07:13:32,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN 2024-11-19T07:13:32,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN 2024-11-19T07:13:32,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN 2024-11-19T07:13:32,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN 2024-11-19T07:13:32,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure e880c03f4c433baa6242dd65ef2d8486, server=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,793 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08f6487d04261d6f0713b4b5adeacf2e, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,793 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66fe58970f9c4a1cba9c3e3da843ed3d, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,793 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3aa6780c71c61c778946d2086357143d, server=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,793 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,793 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7d9d31e1a2abc752a2056ed0e3a3fcec, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,793 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2df93dc30d9c55d765ada1f50e320789, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,793 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,793 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,793 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,793 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,793 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,793 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,793 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,793 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,793 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:32,793 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,793 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:32,793 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,793 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:32,793 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,794 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN 2024-11-19T07:13:32,794 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,794 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN 2024-11-19T07:13:32,793 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:32,794 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,794 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN 2024-11-19T07:13:32,794 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,794 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN 2024-11-19T07:13:32,794 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,794 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN 2024-11-19T07:13:32,794 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure e880c03f4c433baa6242dd65ef2d8486, server=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,794 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08f6487d04261d6f0713b4b5adeacf2e, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,794 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66fe58970f9c4a1cba9c3e3da843ed3d, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,794 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3aa6780c71c61c778946d2086357143d, server=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,794 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,794 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7d9d31e1a2abc752a2056ed0e3a3fcec, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,794 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2df93dc30d9c55d765ada1f50e320789, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,794 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 16 msec 2024-11-19T07:13:32,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=65 2024-11-19T07:13:32,796 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=db0afee3eab9,42435,1732000404708, table=hbase:meta, region=1588230740 2024-11-19T07:13:32,796 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,42435,1732000404708, state=OPEN 2024-11-19T07:13:32,798 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:32,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:32,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:32,807 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:32,807 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:32,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:32,807 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:32,808 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN to state=OFFLINE, location=null, table=test, region=08f6487d04261d6f0713b4b5adeacf2e to restore RIT 2024-11-19T07:13:32,808 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN to state=OFFLINE, location=null, table=test, region=3aa6780c71c61c778946d2086357143d to restore RIT 2024-11-19T07:13:32,809 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN to state=OFFLINE, location=null, table=test, region=2f38687f659a16d9e2a781f2a75d514a to restore RIT 2024-11-19T07:13:32,809 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN to state=OFFLINE, location=null, table=test, region=e880c03f4c433baa6242dd65ef2d8486 to restore RIT 2024-11-19T07:13:32,809 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN to state=OFFLINE, location=null, table=test, region=2df93dc30d9c55d765ada1f50e320789 to restore RIT 2024-11-19T07:13:32,809 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN to state=OFFLINE, location=null, table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d to restore RIT 2024-11-19T07:13:32,809 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN to state=OFFLINE, location=null, table=test, region=bbb518f4693490310eca052bf28f87fc to restore RIT 2024-11-19T07:13:32,809 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN to state=OFFLINE, location=null, table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec to restore RIT 2024-11-19T07:13:32,824 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732000442824 2024-11-19T07:13:32,824 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T07:13:32,826 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:32,828 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T07:13:32,838 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T07:13:32,838 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7d9d31e1a2abc752a2056ed0e3a3fcec, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,838 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:32,839 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7d9d31e1a2abc752a2056ed0e3a3fcec, server=db0afee3eab9,42435,1732000404708 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:32,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02309bab61f6207d73b9f19985e1fdb9, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dcc969f79695b5f0d224b605268d2d, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d9d31e1a2abc752a2056ed0e3a3fcec, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ece759217c055d9a295034ce69dd86d, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e880c03f4c433baa6242dd65ef2d8486, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49b850b07eedb76caa94e4c93ac5c832, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db4ba820395e8b4a2b38d57f8a222e97, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66fe58970f9c4a1cba9c3e3da843ed3d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43676ec232db06c9bf4af64db1ede4f5, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4421c1189c5fdff3a40dcc58728c41ab, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbb518f4693490310eca052bf28f87fc, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82941673e9ad03e5a037091750c3f315, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74657be6232ad22aaca548d82491de41, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=5 2024-11-19T07:13:32,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf73eccbee4f01f5a86b87f17dcdb8b, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a637105eb1d8b75696b3cfabe5d8173, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df93dc30d9c55d765ada1f50e320789, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dba97405d3b35647d42d266df5e81cec, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d703151d82f3a429f348967988283542, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f38687f659a16d9e2a781f2a75d514a, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9357ae6c6e8337f25dacbf0a10701ef6, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08f6487d04261d6f0713b4b5adeacf2e, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=611b07fbb4fb5e3ea7255d9fa0f9fe40, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=126b77a274d2ba63b12dd23c9e7ababb, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3aa6780c71c61c778946d2086357143d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32d987cca115e6f476cdb5686be2b726, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,855 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T07:13:32,855 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732000472855 2024-11-19T07:13:32,855 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732000532855 2024-11-19T07:13:32,855 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 31 msec 2024-11-19T07:13:32,855 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T07:13:32,856 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:32,856 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T07:13:32,856 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T07:13:32,860 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T07:13:32,860 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T07:13:32,860 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T07:13:32,861 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T07:13:32,868 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 7 msec 2024-11-19T07:13:32,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T07:13:32,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:32,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:32,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,882 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,882 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,882 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,882 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,882 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:32,882 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:32,882 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,882 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,882 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T07:13:32,887 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T07:13:32,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, REOPEN/MOVE 2024-11-19T07:13:32,894 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true 2024-11-19T07:13:32,894 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,894 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,894 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T07:13:32,894 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,894 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,894 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,894 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:32,894 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:32,894 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:32,895 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN 2024-11-19T07:13:32,895 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN 2024-11-19T07:13:32,895 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN 2024-11-19T07:13:32,895 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN 2024-11-19T07:13:32,895 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN 2024-11-19T07:13:32,895 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure e880c03f4c433baa6242dd65ef2d8486, server=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,895 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08f6487d04261d6f0713b4b5adeacf2e, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,895 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66fe58970f9c4a1cba9c3e3da843ed3d, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,895 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3aa6780c71c61c778946d2086357143d, server=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,895 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,895 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7d9d31e1a2abc752a2056ed0e3a3fcec, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,895 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,895 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,895 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,895 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,895 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,895 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,895 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,895 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,895 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:32,896 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,896 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:32,896 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,896 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:32,896 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,896 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN 2024-11-19T07:13:32,896 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,896 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN 2024-11-19T07:13:32,896 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,896 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN 2024-11-19T07:13:32,896 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,896 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN 2024-11-19T07:13:32,896 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,896 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN 2024-11-19T07:13:32,896 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure e880c03f4c433baa6242dd65ef2d8486, server=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,896 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08f6487d04261d6f0713b4b5adeacf2e, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,896 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66fe58970f9c4a1cba9c3e3da843ed3d, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,896 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3aa6780c71c61c778946d2086357143d, server=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,896 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,896 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7d9d31e1a2abc752a2056ed0e3a3fcec, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,897 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 28 msec 2024-11-19T07:13:32,898 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:32,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=65 2024-11-19T07:13:32,905 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=db0afee3eab9,42435,1732000404708, table=hbase:meta, region=1588230740 2024-11-19T07:13:32,905 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,42435,1732000404708, state=OPEN 2024-11-19T07:13:32,909 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:32,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:32,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:32,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:32,912 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:32,912 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:32,912 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:32,913 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN to state=OFFLINE, location=null, table=test, region=08f6487d04261d6f0713b4b5adeacf2e to restore RIT 2024-11-19T07:13:32,913 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN to state=OFFLINE, location=null, table=test, region=3aa6780c71c61c778946d2086357143d to restore RIT 2024-11-19T07:13:32,913 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN to state=OFFLINE, location=null, table=test, region=2f38687f659a16d9e2a781f2a75d514a to restore RIT 2024-11-19T07:13:32,913 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN to state=OFFLINE, location=null, table=test, region=e880c03f4c433baa6242dd65ef2d8486 to restore RIT 2024-11-19T07:13:32,913 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN to state=OFFLINE, location=null, table=test, region=2df93dc30d9c55d765ada1f50e320789 to restore RIT 2024-11-19T07:13:32,913 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN to state=OFFLINE, location=null, table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d to restore RIT 2024-11-19T07:13:32,913 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN to state=OFFLINE, location=null, table=test, region=bbb518f4693490310eca052bf28f87fc to restore RIT 2024-11-19T07:13:32,913 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN to state=OFFLINE, location=null, table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec to restore RIT 2024-11-19T07:13:32,916 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732000442916 2024-11-19T07:13:32,916 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T07:13:32,916 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:32,918 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T07:13:32,924 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T07:13:32,924 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:32,925 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:32,927 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02309bab61f6207d73b9f19985e1fdb9, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,928 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dcc969f79695b5f0d224b605268d2d, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,928 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d9d31e1a2abc752a2056ed0e3a3fcec, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,928 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ece759217c055d9a295034ce69dd86d, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,929 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e880c03f4c433baa6242dd65ef2d8486, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,929 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49b850b07eedb76caa94e4c93ac5c832, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,929 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db4ba820395e8b4a2b38d57f8a222e97, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,929 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66fe58970f9c4a1cba9c3e3da843ed3d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43676ec232db06c9bf4af64db1ede4f5, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4421c1189c5fdff3a40dcc58728c41ab, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbb518f4693490310eca052bf28f87fc, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82941673e9ad03e5a037091750c3f315, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74657be6232ad22aaca548d82491de41, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=5 2024-11-19T07:13:32,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf73eccbee4f01f5a86b87f17dcdb8b, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a637105eb1d8b75696b3cfabe5d8173, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df93dc30d9c55d765ada1f50e320789, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dba97405d3b35647d42d266df5e81cec, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d703151d82f3a429f348967988283542, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f38687f659a16d9e2a781f2a75d514a, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9357ae6c6e8337f25dacbf0a10701ef6, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08f6487d04261d6f0713b4b5adeacf2e, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=611b07fbb4fb5e3ea7255d9fa0f9fe40, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=126b77a274d2ba63b12dd23c9e7ababb, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3aa6780c71c61c778946d2086357143d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32d987cca115e6f476cdb5686be2b726, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,933 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T07:13:32,933 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732000472933 2024-11-19T07:13:32,933 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732000532933 2024-11-19T07:13:32,934 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 17 msec 2024-11-19T07:13:32,934 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T07:13:32,934 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:32,934 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T07:13:32,934 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T07:13:32,934 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T07:13:32,934 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T07:13:32,934 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T07:13:32,935 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T07:13:32,936 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T07:13:32,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T07:13:32,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:32,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:32,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,941 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,941 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,941 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,941 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,942 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,942 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,942 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:32,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:32,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T07:13:32,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T07:13:32,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, REOPEN/MOVE 2024-11-19T07:13:32,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true 2024-11-19T07:13:32,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T07:13:32,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:32,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:32,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:32,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN 2024-11-19T07:13:32,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN 2024-11-19T07:13:32,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN 2024-11-19T07:13:32,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN 2024-11-19T07:13:32,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN 2024-11-19T07:13:32,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure e880c03f4c433baa6242dd65ef2d8486, server=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08f6487d04261d6f0713b4b5adeacf2e, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,949 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66fe58970f9c4a1cba9c3e3da843ed3d, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,949 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3aa6780c71c61c778946d2086357143d, server=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,949 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,949 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,949 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,949 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:32,949 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,949 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:32,949 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,949 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:32,949 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,949 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN 2024-11-19T07:13:32,950 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,950 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN 2024-11-19T07:13:32,950 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,950 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN 2024-11-19T07:13:32,950 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,950 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN 2024-11-19T07:13:32,950 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:32,950 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN 2024-11-19T07:13:32,950 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure e880c03f4c433baa6242dd65ef2d8486, server=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,950 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08f6487d04261d6f0713b4b5adeacf2e, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,950 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66fe58970f9c4a1cba9c3e3da843ed3d, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,950 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3aa6780c71c61c778946d2086357143d, server=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,950 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:32,950 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 14 msec 2024-11-19T07:13:32,951 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=65 2024-11-19T07:13:32,951 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=db0afee3eab9,42435,1732000404708, table=hbase:meta, region=1588230740 2024-11-19T07:13:32,951 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,42435,1732000404708, state=OPEN 2024-11-19T07:13:32,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:32,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:32,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:32,964 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:32,964 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:32,964 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:32,965 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN to state=OFFLINE, location=null, table=test, region=08f6487d04261d6f0713b4b5adeacf2e to restore RIT 2024-11-19T07:13:32,965 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN to state=OFFLINE, location=null, table=test, region=3aa6780c71c61c778946d2086357143d to restore RIT 2024-11-19T07:13:32,965 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN to state=OFFLINE, location=null, table=test, region=2f38687f659a16d9e2a781f2a75d514a to restore RIT 2024-11-19T07:13:32,966 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN to state=OFFLINE, location=null, table=test, region=e880c03f4c433baa6242dd65ef2d8486 to restore RIT 2024-11-19T07:13:32,966 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN to state=OFFLINE, location=null, table=test, region=2df93dc30d9c55d765ada1f50e320789 to restore RIT 2024-11-19T07:13:32,966 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN to state=OFFLINE, location=null, table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d to restore RIT 2024-11-19T07:13:32,966 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN to state=OFFLINE, location=null, table=test, region=bbb518f4693490310eca052bf28f87fc to restore RIT 2024-11-19T07:13:32,966 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN to state=OFFLINE, location=null, table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec to restore RIT 2024-11-19T07:13:32,966 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732000442966 2024-11-19T07:13:32,966 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T07:13:32,966 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:32,969 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T07:13:32,978 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T07:13:32,978 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3aa6780c71c61c778946d2086357143d, server=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:32,978 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3aa6780c71c61c778946d2086357143d, server=db0afee3eab9,34425,1732000404604 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:32,979 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02309bab61f6207d73b9f19985e1fdb9, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dcc969f79695b5f0d224b605268d2d, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d9d31e1a2abc752a2056ed0e3a3fcec, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ece759217c055d9a295034ce69dd86d, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e880c03f4c433baa6242dd65ef2d8486, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49b850b07eedb76caa94e4c93ac5c832, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db4ba820395e8b4a2b38d57f8a222e97, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,983 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66fe58970f9c4a1cba9c3e3da843ed3d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,983 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43676ec232db06c9bf4af64db1ede4f5, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,983 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4421c1189c5fdff3a40dcc58728c41ab, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,984 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbb518f4693490310eca052bf28f87fc, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,984 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82941673e9ad03e5a037091750c3f315, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,984 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,985 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74657be6232ad22aaca548d82491de41, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=5 2024-11-19T07:13:32,985 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf73eccbee4f01f5a86b87f17dcdb8b, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,985 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a637105eb1d8b75696b3cfabe5d8173, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,985 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df93dc30d9c55d765ada1f50e320789, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dba97405d3b35647d42d266df5e81cec, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d703151d82f3a429f348967988283542, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f38687f659a16d9e2a781f2a75d514a, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9357ae6c6e8337f25dacbf0a10701ef6, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08f6487d04261d6f0713b4b5adeacf2e, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=611b07fbb4fb5e3ea7255d9fa0f9fe40, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=126b77a274d2ba63b12dd23c9e7ababb, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3aa6780c71c61c778946d2086357143d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:32,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32d987cca115e6f476cdb5686be2b726, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:32,988 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T07:13:32,988 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732000472988 2024-11-19T07:13:32,988 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732000532988 2024-11-19T07:13:32,988 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 21 msec 2024-11-19T07:13:32,988 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T07:13:32,988 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:32,988 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T07:13:32,988 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T07:13:32,989 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T07:13:32,989 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T07:13:32,989 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T07:13:32,989 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T07:13:32,990 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T07:13:32,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T07:13:32,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:32,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:32,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,996 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,996 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:32,996 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:32,996 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:32,996 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:32,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T07:13:32,997 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T07:13:32,997 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, REOPEN/MOVE 2024-11-19T07:13:32,999 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true 2024-11-19T07:13:32,999 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,999 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:32,999 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T07:13:32,999 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,999 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,999 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:32,999 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:32,999 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:32,999 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:33,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN 2024-11-19T07:13:33,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN 2024-11-19T07:13:33,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN 2024-11-19T07:13:33,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN 2024-11-19T07:13:33,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN 2024-11-19T07:13:33,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure e880c03f4c433baa6242dd65ef2d8486, server=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08f6487d04261d6f0713b4b5adeacf2e, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66fe58970f9c4a1cba9c3e3da843ed3d, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3aa6780c71c61c778946d2086357143d, server=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,000 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,000 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,000 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,000 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,000 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,000 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,000 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,000 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,001 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:33,001 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,001 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:33,001 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,001 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:33,001 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,001 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN 2024-11-19T07:13:33,001 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,001 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN 2024-11-19T07:13:33,001 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,001 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN 2024-11-19T07:13:33,001 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,001 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN 2024-11-19T07:13:33,001 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,001 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN 2024-11-19T07:13:33,001 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure e880c03f4c433baa6242dd65ef2d8486, server=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,001 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08f6487d04261d6f0713b4b5adeacf2e, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,001 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66fe58970f9c4a1cba9c3e3da843ed3d, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,001 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3aa6780c71c61c778946d2086357143d, server=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,001 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-19T07:13:33,002 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=65 2024-11-19T07:13:33,003 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=db0afee3eab9,42435,1732000404708, table=hbase:meta, region=1588230740 2024-11-19T07:13:33,003 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,42435,1732000404708, state=OPEN 2024-11-19T07:13:33,013 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,016 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,016 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,016 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,017 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN to state=OFFLINE, location=null, table=test, region=08f6487d04261d6f0713b4b5adeacf2e to restore RIT 2024-11-19T07:13:33,017 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN to state=OFFLINE, location=null, table=test, region=3aa6780c71c61c778946d2086357143d to restore RIT 2024-11-19T07:13:33,017 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN to state=OFFLINE, location=null, table=test, region=2f38687f659a16d9e2a781f2a75d514a to restore RIT 2024-11-19T07:13:33,017 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN to state=OFFLINE, location=null, table=test, region=e880c03f4c433baa6242dd65ef2d8486 to restore RIT 2024-11-19T07:13:33,017 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN to state=OFFLINE, location=null, table=test, region=2df93dc30d9c55d765ada1f50e320789 to restore RIT 2024-11-19T07:13:33,017 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN to state=OFFLINE, location=null, table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d to restore RIT 2024-11-19T07:13:33,017 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN to state=OFFLINE, location=null, table=test, region=bbb518f4693490310eca052bf28f87fc to restore RIT 2024-11-19T07:13:33,017 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN to state=OFFLINE, location=null, table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec to restore RIT 2024-11-19T07:13:33,018 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732000443018 2024-11-19T07:13:33,018 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T07:13:33,018 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,020 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T07:13:33,021 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T07:13:33,021 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66fe58970f9c4a1cba9c3e3da843ed3d, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,021 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66fe58970f9c4a1cba9c3e3da843ed3d, server=db0afee3eab9,42435,1732000404708 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02309bab61f6207d73b9f19985e1fdb9, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dcc969f79695b5f0d224b605268d2d, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d9d31e1a2abc752a2056ed0e3a3fcec, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ece759217c055d9a295034ce69dd86d, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e880c03f4c433baa6242dd65ef2d8486, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49b850b07eedb76caa94e4c93ac5c832, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db4ba820395e8b4a2b38d57f8a222e97, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66fe58970f9c4a1cba9c3e3da843ed3d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43676ec232db06c9bf4af64db1ede4f5, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4421c1189c5fdff3a40dcc58728c41ab, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbb518f4693490310eca052bf28f87fc, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82941673e9ad03e5a037091750c3f315, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74657be6232ad22aaca548d82491de41, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=5 2024-11-19T07:13:33,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf73eccbee4f01f5a86b87f17dcdb8b, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a637105eb1d8b75696b3cfabe5d8173, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df93dc30d9c55d765ada1f50e320789, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dba97405d3b35647d42d266df5e81cec, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d703151d82f3a429f348967988283542, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f38687f659a16d9e2a781f2a75d514a, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9357ae6c6e8337f25dacbf0a10701ef6, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08f6487d04261d6f0713b4b5adeacf2e, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=611b07fbb4fb5e3ea7255d9fa0f9fe40, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=126b77a274d2ba63b12dd23c9e7ababb, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3aa6780c71c61c778946d2086357143d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32d987cca115e6f476cdb5686be2b726, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,029 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T07:13:33,029 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732000473029 2024-11-19T07:13:33,029 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732000533029 2024-11-19T07:13:33,029 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-19T07:13:33,029 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T07:13:33,029 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,029 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T07:13:33,029 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T07:13:33,029 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T07:13:33,029 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T07:13:33,029 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T07:13:33,030 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T07:13:33,030 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T07:13:33,033 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T07:13:33,033 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,033 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:33,033 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,033 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:33,033 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,033 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,033 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,033 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:33,033 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,033 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,033 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:33,033 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,033 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,033 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,033 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:33,033 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,033 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:33,033 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T07:13:33,034 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T07:13:33,034 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, REOPEN/MOVE 2024-11-19T07:13:33,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true 2024-11-19T07:13:33,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,036 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T07:13:33,036 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,036 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,036 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,036 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:33,036 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:33,036 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:33,036 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN 2024-11-19T07:13:33,036 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN 2024-11-19T07:13:33,036 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN 2024-11-19T07:13:33,036 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN 2024-11-19T07:13:33,036 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN 2024-11-19T07:13:33,036 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure e880c03f4c433baa6242dd65ef2d8486, server=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,036 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08f6487d04261d6f0713b4b5adeacf2e, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,036 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66fe58970f9c4a1cba9c3e3da843ed3d, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,036 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,036 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,036 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,036 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,036 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,036 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,036 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,036 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,036 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:33,036 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,036 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:33,036 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,036 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:33,036 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,036 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN 2024-11-19T07:13:33,036 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,037 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN 2024-11-19T07:13:33,037 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,037 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN 2024-11-19T07:13:33,037 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,037 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN 2024-11-19T07:13:33,037 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,037 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN 2024-11-19T07:13:33,037 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure e880c03f4c433baa6242dd65ef2d8486, server=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,037 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08f6487d04261d6f0713b4b5adeacf2e, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,037 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66fe58970f9c4a1cba9c3e3da843ed3d, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,037 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-19T07:13:33,038 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=65 2024-11-19T07:13:33,038 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=db0afee3eab9,42435,1732000404708, table=hbase:meta, region=1588230740 2024-11-19T07:13:33,038 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,42435,1732000404708, state=OPEN 2024-11-19T07:13:33,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,048 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,048 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,048 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,049 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN to state=OFFLINE, location=null, table=test, region=08f6487d04261d6f0713b4b5adeacf2e to restore RIT 2024-11-19T07:13:33,049 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN to state=OFFLINE, location=null, table=test, region=3aa6780c71c61c778946d2086357143d to restore RIT 2024-11-19T07:13:33,049 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN to state=OFFLINE, location=null, table=test, region=2f38687f659a16d9e2a781f2a75d514a to restore RIT 2024-11-19T07:13:33,049 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN to state=OFFLINE, location=null, table=test, region=e880c03f4c433baa6242dd65ef2d8486 to restore RIT 2024-11-19T07:13:33,049 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN to state=OFFLINE, location=null, table=test, region=2df93dc30d9c55d765ada1f50e320789 to restore RIT 2024-11-19T07:13:33,049 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN to state=OFFLINE, location=null, table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d to restore RIT 2024-11-19T07:13:33,049 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN to state=OFFLINE, location=null, table=test, region=bbb518f4693490310eca052bf28f87fc to restore RIT 2024-11-19T07:13:33,049 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN to state=OFFLINE, location=null, table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec to restore RIT 2024-11-19T07:13:33,050 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732000443050 2024-11-19T07:13:33,050 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T07:13:33,050 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,051 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T07:13:33,054 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T07:13:33,054 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08f6487d04261d6f0713b4b5adeacf2e, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,054 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08f6487d04261d6f0713b4b5adeacf2e, server=db0afee3eab9,42435,1732000404708 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02309bab61f6207d73b9f19985e1fdb9, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dcc969f79695b5f0d224b605268d2d, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d9d31e1a2abc752a2056ed0e3a3fcec, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ece759217c055d9a295034ce69dd86d, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e880c03f4c433baa6242dd65ef2d8486, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49b850b07eedb76caa94e4c93ac5c832, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db4ba820395e8b4a2b38d57f8a222e97, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66fe58970f9c4a1cba9c3e3da843ed3d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43676ec232db06c9bf4af64db1ede4f5, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4421c1189c5fdff3a40dcc58728c41ab, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbb518f4693490310eca052bf28f87fc, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82941673e9ad03e5a037091750c3f315, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74657be6232ad22aaca548d82491de41, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=5 2024-11-19T07:13:33,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf73eccbee4f01f5a86b87f17dcdb8b, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a637105eb1d8b75696b3cfabe5d8173, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df93dc30d9c55d765ada1f50e320789, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dba97405d3b35647d42d266df5e81cec, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,059 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d703151d82f3a429f348967988283542, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,059 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f38687f659a16d9e2a781f2a75d514a, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,059 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9357ae6c6e8337f25dacbf0a10701ef6, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,059 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08f6487d04261d6f0713b4b5adeacf2e, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,059 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=611b07fbb4fb5e3ea7255d9fa0f9fe40, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,059 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=126b77a274d2ba63b12dd23c9e7ababb, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,059 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3aa6780c71c61c778946d2086357143d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,059 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32d987cca115e6f476cdb5686be2b726, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,060 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T07:13:33,060 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732000473060 2024-11-19T07:13:33,060 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732000533060 2024-11-19T07:13:33,060 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-19T07:13:33,060 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T07:13:33,060 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,060 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T07:13:33,060 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T07:13:33,060 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T07:13:33,060 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T07:13:33,060 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T07:13:33,060 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T07:13:33,061 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T07:13:33,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T07:13:33,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:33,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:33,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,064 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:33,064 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,064 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,064 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,064 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,064 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,064 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:33,064 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,064 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:33,064 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T07:13:33,064 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T07:13:33,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, REOPEN/MOVE 2024-11-19T07:13:33,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true 2024-11-19T07:13:33,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T07:13:33,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:33,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:33,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:33,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN 2024-11-19T07:13:33,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN 2024-11-19T07:13:33,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN 2024-11-19T07:13:33,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN 2024-11-19T07:13:33,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN 2024-11-19T07:13:33,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure e880c03f4c433baa6242dd65ef2d8486, server=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08f6487d04261d6f0713b4b5adeacf2e, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,067 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,067 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,067 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:33,067 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,067 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:33,067 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,067 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:33,067 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,067 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN 2024-11-19T07:13:33,067 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,068 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN 2024-11-19T07:13:33,068 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,068 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN 2024-11-19T07:13:33,068 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,068 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN 2024-11-19T07:13:33,068 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,068 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN 2024-11-19T07:13:33,068 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure e880c03f4c433baa6242dd65ef2d8486, server=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,068 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08f6487d04261d6f0713b4b5adeacf2e, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,068 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-19T07:13:33,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=65 2024-11-19T07:13:33,069 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=db0afee3eab9,42435,1732000404708, table=hbase:meta, region=1588230740 2024-11-19T07:13:33,070 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,42435,1732000404708, state=OPEN 2024-11-19T07:13:33,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,080 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,080 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,080 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,081 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN to state=OFFLINE, location=null, table=test, region=08f6487d04261d6f0713b4b5adeacf2e to restore RIT 2024-11-19T07:13:33,081 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN to state=OFFLINE, location=null, table=test, region=3aa6780c71c61c778946d2086357143d to restore RIT 2024-11-19T07:13:33,081 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN to state=OFFLINE, location=null, table=test, region=2f38687f659a16d9e2a781f2a75d514a to restore RIT 2024-11-19T07:13:33,081 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN to state=OFFLINE, location=null, table=test, region=e880c03f4c433baa6242dd65ef2d8486 to restore RIT 2024-11-19T07:13:33,081 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN to state=OFFLINE, location=null, table=test, region=2df93dc30d9c55d765ada1f50e320789 to restore RIT 2024-11-19T07:13:33,081 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN to state=OFFLINE, location=null, table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d to restore RIT 2024-11-19T07:13:33,081 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN to state=OFFLINE, location=null, table=test, region=bbb518f4693490310eca052bf28f87fc to restore RIT 2024-11-19T07:13:33,081 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN to state=OFFLINE, location=null, table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec to restore RIT 2024-11-19T07:13:33,082 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732000443082 2024-11-19T07:13:33,082 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T07:13:33,083 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,085 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T07:13:33,086 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T07:13:33,086 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure e880c03f4c433baa6242dd65ef2d8486, server=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,086 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure e880c03f4c433baa6242dd65ef2d8486, server=db0afee3eab9,34425,1732000404604 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02309bab61f6207d73b9f19985e1fdb9, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dcc969f79695b5f0d224b605268d2d, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d9d31e1a2abc752a2056ed0e3a3fcec, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ece759217c055d9a295034ce69dd86d, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e880c03f4c433baa6242dd65ef2d8486, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49b850b07eedb76caa94e4c93ac5c832, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db4ba820395e8b4a2b38d57f8a222e97, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66fe58970f9c4a1cba9c3e3da843ed3d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43676ec232db06c9bf4af64db1ede4f5, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,094 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4421c1189c5fdff3a40dcc58728c41ab, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,094 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbb518f4693490310eca052bf28f87fc, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,094 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82941673e9ad03e5a037091750c3f315, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,094 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,094 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74657be6232ad22aaca548d82491de41, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=5 2024-11-19T07:13:33,094 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf73eccbee4f01f5a86b87f17dcdb8b, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,095 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a637105eb1d8b75696b3cfabe5d8173, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,095 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df93dc30d9c55d765ada1f50e320789, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,095 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dba97405d3b35647d42d266df5e81cec, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,095 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d703151d82f3a429f348967988283542, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,095 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f38687f659a16d9e2a781f2a75d514a, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,095 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9357ae6c6e8337f25dacbf0a10701ef6, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08f6487d04261d6f0713b4b5adeacf2e, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=611b07fbb4fb5e3ea7255d9fa0f9fe40, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=126b77a274d2ba63b12dd23c9e7ababb, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3aa6780c71c61c778946d2086357143d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32d987cca115e6f476cdb5686be2b726, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,096 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T07:13:33,096 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732000473096 2024-11-19T07:13:33,096 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732000533096 2024-11-19T07:13:33,096 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 14 msec 2024-11-19T07:13:33,097 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T07:13:33,097 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,097 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T07:13:33,097 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T07:13:33,097 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T07:13:33,097 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T07:13:33,097 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T07:13:33,098 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T07:13:33,099 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T07:13:33,101 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T07:13:33,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:33,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:33,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:33,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:33,103 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T07:13:33,104 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T07:13:33,104 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, REOPEN/MOVE 2024-11-19T07:13:33,105 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true 2024-11-19T07:13:33,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T07:13:33,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:33,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:33,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:33,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN 2024-11-19T07:13:33,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN 2024-11-19T07:13:33,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN 2024-11-19T07:13:33,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN 2024-11-19T07:13:33,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN 2024-11-19T07:13:33,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure e880c03f4c433baa6242dd65ef2d8486, server=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,108 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,108 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,108 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,108 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,108 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,108 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,108 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,108 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,108 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:33,108 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,108 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:33,108 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,108 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:33,108 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,108 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN 2024-11-19T07:13:33,108 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,108 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN 2024-11-19T07:13:33,108 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,108 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN 2024-11-19T07:13:33,108 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,108 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN 2024-11-19T07:13:33,109 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,109 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN 2024-11-19T07:13:33,109 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure e880c03f4c433baa6242dd65ef2d8486, server=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,109 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-19T07:13:33,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=65 2024-11-19T07:13:33,110 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=db0afee3eab9,42435,1732000404708, table=hbase:meta, region=1588230740 2024-11-19T07:13:33,110 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,42435,1732000404708, state=OPEN 2024-11-19T07:13:33,117 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,122 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,122 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,122 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,123 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN to state=OFFLINE, location=null, table=test, region=08f6487d04261d6f0713b4b5adeacf2e to restore RIT 2024-11-19T07:13:33,123 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN to state=OFFLINE, location=null, table=test, region=3aa6780c71c61c778946d2086357143d to restore RIT 2024-11-19T07:13:33,123 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN to state=OFFLINE, location=null, table=test, region=2f38687f659a16d9e2a781f2a75d514a to restore RIT 2024-11-19T07:13:33,123 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN to state=OFFLINE, location=null, table=test, region=e880c03f4c433baa6242dd65ef2d8486 to restore RIT 2024-11-19T07:13:33,124 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN to state=OFFLINE, location=null, table=test, region=2df93dc30d9c55d765ada1f50e320789 to restore RIT 2024-11-19T07:13:33,124 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN to state=OFFLINE, location=null, table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d to restore RIT 2024-11-19T07:13:33,124 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN to state=OFFLINE, location=null, table=test, region=bbb518f4693490310eca052bf28f87fc to restore RIT 2024-11-19T07:13:33,124 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN to state=OFFLINE, location=null, table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec to restore RIT 2024-11-19T07:13:33,125 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732000443125 2024-11-19T07:13:33,125 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T07:13:33,125 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,127 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T07:13:33,132 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T07:13:33,132 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN 2024-11-19T07:13:33,132 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02309bab61f6207d73b9f19985e1fdb9, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,132 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dcc969f79695b5f0d224b605268d2d, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d9d31e1a2abc752a2056ed0e3a3fcec, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ece759217c055d9a295034ce69dd86d, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e880c03f4c433baa6242dd65ef2d8486, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49b850b07eedb76caa94e4c93ac5c832, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db4ba820395e8b4a2b38d57f8a222e97, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66fe58970f9c4a1cba9c3e3da843ed3d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43676ec232db06c9bf4af64db1ede4f5, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4421c1189c5fdff3a40dcc58728c41ab, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbb518f4693490310eca052bf28f87fc, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82941673e9ad03e5a037091750c3f315, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74657be6232ad22aaca548d82491de41, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=5 2024-11-19T07:13:33,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf73eccbee4f01f5a86b87f17dcdb8b, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a637105eb1d8b75696b3cfabe5d8173, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df93dc30d9c55d765ada1f50e320789, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dba97405d3b35647d42d266df5e81cec, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d703151d82f3a429f348967988283542, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f38687f659a16d9e2a781f2a75d514a, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9357ae6c6e8337f25dacbf0a10701ef6, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08f6487d04261d6f0713b4b5adeacf2e, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=611b07fbb4fb5e3ea7255d9fa0f9fe40, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,137 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=126b77a274d2ba63b12dd23c9e7ababb, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,137 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3aa6780c71c61c778946d2086357143d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,137 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32d987cca115e6f476cdb5686be2b726, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,137 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T07:13:33,137 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732000473137 2024-11-19T07:13:33,137 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732000533137 2024-11-19T07:13:33,137 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 12 msec 2024-11-19T07:13:33,137 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T07:13:33,137 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,137 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T07:13:33,138 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T07:13:33,138 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T07:13:33,138 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T07:13:33,138 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T07:13:33,138 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T07:13:33,139 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T07:13:33,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T07:13:33,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:33,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:33,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:33,141 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T07:13:33,142 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T07:13:33,142 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, REOPEN/MOVE 2024-11-19T07:13:33,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true 2024-11-19T07:13:33,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T07:13:33,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:33,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:33,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:33,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN 2024-11-19T07:13:33,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN 2024-11-19T07:13:33,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN 2024-11-19T07:13:33,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN 2024-11-19T07:13:33,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN 2024-11-19T07:13:33,144 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,144 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,144 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,144 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,144 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,144 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,144 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,144 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,144 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:33,144 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,144 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:33,144 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,144 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:33,144 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,144 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN 2024-11-19T07:13:33,144 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,144 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN 2024-11-19T07:13:33,144 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,144 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN 2024-11-19T07:13:33,144 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,144 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN 2024-11-19T07:13:33,144 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,144 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN 2024-11-19T07:13:33,145 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-19T07:13:33,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=65 2024-11-19T07:13:33,146 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=db0afee3eab9,42435,1732000404708, table=hbase:meta, region=1588230740 2024-11-19T07:13:33,146 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,42435,1732000404708, state=OPEN 2024-11-19T07:13:33,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,154 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,154 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,154 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,154 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN to state=OFFLINE, location=null, table=test, region=08f6487d04261d6f0713b4b5adeacf2e to restore RIT 2024-11-19T07:13:33,155 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN to state=OFFLINE, location=null, table=test, region=3aa6780c71c61c778946d2086357143d to restore RIT 2024-11-19T07:13:33,155 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN to state=OFFLINE, location=null, table=test, region=2f38687f659a16d9e2a781f2a75d514a to restore RIT 2024-11-19T07:13:33,155 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN to state=OFFLINE, location=null, table=test, region=e880c03f4c433baa6242dd65ef2d8486 to restore RIT 2024-11-19T07:13:33,155 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN to state=OFFLINE, location=null, table=test, region=2df93dc30d9c55d765ada1f50e320789 to restore RIT 2024-11-19T07:13:33,155 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN to state=OFFLINE, location=null, table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d to restore RIT 2024-11-19T07:13:33,155 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN to state=OFFLINE, location=null, table=test, region=bbb518f4693490310eca052bf28f87fc to restore RIT 2024-11-19T07:13:33,155 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec, ASSIGN to state=OFFLINE, location=null, table=test, region=7d9d31e1a2abc752a2056ed0e3a3fcec to restore RIT 2024-11-19T07:13:33,155 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732000443155 2024-11-19T07:13:33,155 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T07:13:33,155 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,157 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T07:13:33,158 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T07:13:33,158 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN 2024-11-19T07:13:33,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,158 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02309bab61f6207d73b9f19985e1fdb9, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dcc969f79695b5f0d224b605268d2d, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d9d31e1a2abc752a2056ed0e3a3fcec, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ece759217c055d9a295034ce69dd86d, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e880c03f4c433baa6242dd65ef2d8486, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49b850b07eedb76caa94e4c93ac5c832, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db4ba820395e8b4a2b38d57f8a222e97, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66fe58970f9c4a1cba9c3e3da843ed3d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43676ec232db06c9bf4af64db1ede4f5, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4421c1189c5fdff3a40dcc58728c41ab, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbb518f4693490310eca052bf28f87fc, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,163 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82941673e9ad03e5a037091750c3f315, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,163 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,163 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74657be6232ad22aaca548d82491de41, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=5 2024-11-19T07:13:33,163 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf73eccbee4f01f5a86b87f17dcdb8b, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,163 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a637105eb1d8b75696b3cfabe5d8173, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,163 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df93dc30d9c55d765ada1f50e320789, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,163 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dba97405d3b35647d42d266df5e81cec, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,163 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d703151d82f3a429f348967988283542, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,163 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f38687f659a16d9e2a781f2a75d514a, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,164 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9357ae6c6e8337f25dacbf0a10701ef6, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,164 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08f6487d04261d6f0713b4b5adeacf2e, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,164 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=611b07fbb4fb5e3ea7255d9fa0f9fe40, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,164 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=126b77a274d2ba63b12dd23c9e7ababb, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,164 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3aa6780c71c61c778946d2086357143d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,164 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32d987cca115e6f476cdb5686be2b726, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,164 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T07:13:33,164 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732000473164 2024-11-19T07:13:33,164 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732000533164 2024-11-19T07:13:33,164 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-19T07:13:33,164 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T07:13:33,164 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,165 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T07:13:33,165 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T07:13:33,165 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T07:13:33,165 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T07:13:33,165 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T07:13:33,165 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T07:13:33,166 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T07:13:33,167 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T07:13:33,168 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,168 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:33,168 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,168 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,168 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,168 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,168 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,168 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,168 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,168 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,168 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:33,168 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,168 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:33,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T07:13:33,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T07:13:33,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, REOPEN/MOVE 2024-11-19T07:13:33,170 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true 2024-11-19T07:13:33,170 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,170 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,170 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T07:13:33,170 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,170 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,170 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,170 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:33,170 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:33,170 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:33,170 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN 2024-11-19T07:13:33,170 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN 2024-11-19T07:13:33,170 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN 2024-11-19T07:13:33,170 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN 2024-11-19T07:13:33,170 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,170 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,170 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,170 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,170 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,170 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,170 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,170 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,170 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:33,170 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,171 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:33,171 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,171 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:33,171 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,171 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN 2024-11-19T07:13:33,171 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,171 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN 2024-11-19T07:13:33,171 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,171 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN 2024-11-19T07:13:33,171 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,171 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN 2024-11-19T07:13:33,171 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-19T07:13:33,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=65 2024-11-19T07:13:33,172 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=db0afee3eab9,42435,1732000404708, table=hbase:meta, region=1588230740 2024-11-19T07:13:33,172 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,42435,1732000404708, state=OPEN 2024-11-19T07:13:33,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,185 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,185 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,185 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,186 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN to state=OFFLINE, location=null, table=test, region=08f6487d04261d6f0713b4b5adeacf2e to restore RIT 2024-11-19T07:13:33,187 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN to state=OFFLINE, location=null, table=test, region=3aa6780c71c61c778946d2086357143d to restore RIT 2024-11-19T07:13:33,187 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN to state=OFFLINE, location=null, table=test, region=2f38687f659a16d9e2a781f2a75d514a to restore RIT 2024-11-19T07:13:33,187 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN to state=OFFLINE, location=null, table=test, region=e880c03f4c433baa6242dd65ef2d8486 to restore RIT 2024-11-19T07:13:33,187 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN to state=OFFLINE, location=null, table=test, region=2df93dc30d9c55d765ada1f50e320789 to restore RIT 2024-11-19T07:13:33,187 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN to state=OFFLINE, location=null, table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d to restore RIT 2024-11-19T07:13:33,187 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbb518f4693490310eca052bf28f87fc, ASSIGN to state=OFFLINE, location=null, table=test, region=bbb518f4693490310eca052bf28f87fc to restore RIT 2024-11-19T07:13:33,187 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732000443187 2024-11-19T07:13:33,187 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T07:13:33,187 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,189 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T07:13:33,190 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T07:13:33,191 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN 2024-11-19T07:13:33,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,191 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02309bab61f6207d73b9f19985e1fdb9, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dcc969f79695b5f0d224b605268d2d, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d9d31e1a2abc752a2056ed0e3a3fcec, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ece759217c055d9a295034ce69dd86d, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e880c03f4c433baa6242dd65ef2d8486, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49b850b07eedb76caa94e4c93ac5c832, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db4ba820395e8b4a2b38d57f8a222e97, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66fe58970f9c4a1cba9c3e3da843ed3d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43676ec232db06c9bf4af64db1ede4f5, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4421c1189c5fdff3a40dcc58728c41ab, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbb518f4693490310eca052bf28f87fc, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82941673e9ad03e5a037091750c3f315, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74657be6232ad22aaca548d82491de41, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=5 2024-11-19T07:13:33,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf73eccbee4f01f5a86b87f17dcdb8b, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a637105eb1d8b75696b3cfabe5d8173, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df93dc30d9c55d765ada1f50e320789, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dba97405d3b35647d42d266df5e81cec, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d703151d82f3a429f348967988283542, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f38687f659a16d9e2a781f2a75d514a, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9357ae6c6e8337f25dacbf0a10701ef6, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08f6487d04261d6f0713b4b5adeacf2e, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=611b07fbb4fb5e3ea7255d9fa0f9fe40, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=126b77a274d2ba63b12dd23c9e7ababb, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3aa6780c71c61c778946d2086357143d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32d987cca115e6f476cdb5686be2b726, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,198 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T07:13:33,198 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732000473198 2024-11-19T07:13:33,198 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732000533198 2024-11-19T07:13:33,198 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-19T07:13:33,198 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T07:13:33,198 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,199 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T07:13:33,199 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T07:13:33,199 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T07:13:33,199 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T07:13:33,199 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T07:13:33,199 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T07:13:33,200 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T07:13:33,202 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T07:13:33,202 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,202 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:33,202 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,202 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,202 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,202 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,202 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,202 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,202 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,202 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:33,202 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,202 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:33,202 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T07:13:33,203 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T07:13:33,203 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, REOPEN/MOVE 2024-11-19T07:13:33,204 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true 2024-11-19T07:13:33,204 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,204 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,204 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T07:13:33,204 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,204 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,204 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,204 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:33,204 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:33,204 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:33,204 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN 2024-11-19T07:13:33,204 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN 2024-11-19T07:13:33,204 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN 2024-11-19T07:13:33,204 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,205 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,205 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,205 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,205 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,205 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,205 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,205 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,205 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:33,205 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,205 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:33,205 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,205 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:33,205 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,205 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN 2024-11-19T07:13:33,205 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,205 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN 2024-11-19T07:13:33,205 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,205 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN 2024-11-19T07:13:33,205 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-19T07:13:33,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=65 2024-11-19T07:13:33,206 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=db0afee3eab9,42435,1732000404708, table=hbase:meta, region=1588230740 2024-11-19T07:13:33,206 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,42435,1732000404708, state=OPEN 2024-11-19T07:13:33,209 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,217 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,217 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,217 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,218 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN to state=OFFLINE, location=null, table=test, region=08f6487d04261d6f0713b4b5adeacf2e to restore RIT 2024-11-19T07:13:33,218 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN to state=OFFLINE, location=null, table=test, region=3aa6780c71c61c778946d2086357143d to restore RIT 2024-11-19T07:13:33,218 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN to state=OFFLINE, location=null, table=test, region=2f38687f659a16d9e2a781f2a75d514a to restore RIT 2024-11-19T07:13:33,218 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN to state=OFFLINE, location=null, table=test, region=e880c03f4c433baa6242dd65ef2d8486 to restore RIT 2024-11-19T07:13:33,218 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN to state=OFFLINE, location=null, table=test, region=2df93dc30d9c55d765ada1f50e320789 to restore RIT 2024-11-19T07:13:33,218 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d, ASSIGN to state=OFFLINE, location=null, table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d to restore RIT 2024-11-19T07:13:33,219 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732000443219 2024-11-19T07:13:33,219 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T07:13:33,219 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,221 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T07:13:33,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02309bab61f6207d73b9f19985e1fdb9, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,225 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T07:13:33,226 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dcc969f79695b5f0d224b605268d2d, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,226 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN 2024-11-19T07:13:33,226 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,226 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d9d31e1a2abc752a2056ed0e3a3fcec, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,226 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ece759217c055d9a295034ce69dd86d, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,226 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,226 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e880c03f4c433baa6242dd65ef2d8486, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,226 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49b850b07eedb76caa94e4c93ac5c832, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,227 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db4ba820395e8b4a2b38d57f8a222e97, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,227 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66fe58970f9c4a1cba9c3e3da843ed3d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,227 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43676ec232db06c9bf4af64db1ede4f5, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,227 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4421c1189c5fdff3a40dcc58728c41ab, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,227 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbb518f4693490310eca052bf28f87fc, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,227 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82941673e9ad03e5a037091750c3f315, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74657be6232ad22aaca548d82491de41, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=5 2024-11-19T07:13:33,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf73eccbee4f01f5a86b87f17dcdb8b, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a637105eb1d8b75696b3cfabe5d8173, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df93dc30d9c55d765ada1f50e320789, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dba97405d3b35647d42d266df5e81cec, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d703151d82f3a429f348967988283542, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f38687f659a16d9e2a781f2a75d514a, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,229 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9357ae6c6e8337f25dacbf0a10701ef6, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,229 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08f6487d04261d6f0713b4b5adeacf2e, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,229 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=611b07fbb4fb5e3ea7255d9fa0f9fe40, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,229 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=126b77a274d2ba63b12dd23c9e7ababb, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,229 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3aa6780c71c61c778946d2086357143d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,229 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32d987cca115e6f476cdb5686be2b726, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,229 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T07:13:33,230 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732000473229 2024-11-19T07:13:33,230 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732000533230 2024-11-19T07:13:33,230 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-19T07:13:33,230 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T07:13:33,230 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,230 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T07:13:33,230 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T07:13:33,230 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T07:13:33,230 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T07:13:33,230 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T07:13:33,231 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T07:13:33,231 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T07:13:33,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T07:13:33,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:33,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:33,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:33,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T07:13:33,235 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T07:13:33,235 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, REOPEN/MOVE 2024-11-19T07:13:33,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true 2024-11-19T07:13:33,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T07:13:33,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:33,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:33,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:33,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN 2024-11-19T07:13:33,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN 2024-11-19T07:13:33,237 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,238 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,238 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,238 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,238 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,238 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,238 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,238 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,238 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:33,238 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,238 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:33,238 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,238 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:33,238 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,238 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN 2024-11-19T07:13:33,238 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,238 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN 2024-11-19T07:13:33,238 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-19T07:13:33,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=65 2024-11-19T07:13:33,240 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=db0afee3eab9,42435,1732000404708, table=hbase:meta, region=1588230740 2024-11-19T07:13:33,240 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,42435,1732000404708, state=OPEN 2024-11-19T07:13:33,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,248 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,248 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,248 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,249 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN to state=OFFLINE, location=null, table=test, region=08f6487d04261d6f0713b4b5adeacf2e to restore RIT 2024-11-19T07:13:33,250 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN to state=OFFLINE, location=null, table=test, region=3aa6780c71c61c778946d2086357143d to restore RIT 2024-11-19T07:13:33,250 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN to state=OFFLINE, location=null, table=test, region=2f38687f659a16d9e2a781f2a75d514a to restore RIT 2024-11-19T07:13:33,250 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN to state=OFFLINE, location=null, table=test, region=e880c03f4c433baa6242dd65ef2d8486 to restore RIT 2024-11-19T07:13:33,250 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2df93dc30d9c55d765ada1f50e320789, ASSIGN to state=OFFLINE, location=null, table=test, region=2df93dc30d9c55d765ada1f50e320789 to restore RIT 2024-11-19T07:13:33,250 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732000443250 2024-11-19T07:13:33,250 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T07:13:33,250 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,252 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T07:13:33,253 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T07:13:33,254 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN 2024-11-19T07:13:33,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,254 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,257 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02309bab61f6207d73b9f19985e1fdb9, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,257 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dcc969f79695b5f0d224b605268d2d, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,258 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d9d31e1a2abc752a2056ed0e3a3fcec, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,258 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ece759217c055d9a295034ce69dd86d, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,258 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e880c03f4c433baa6242dd65ef2d8486, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,258 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49b850b07eedb76caa94e4c93ac5c832, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,258 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db4ba820395e8b4a2b38d57f8a222e97, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,258 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66fe58970f9c4a1cba9c3e3da843ed3d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43676ec232db06c9bf4af64db1ede4f5, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4421c1189c5fdff3a40dcc58728c41ab, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbb518f4693490310eca052bf28f87fc, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82941673e9ad03e5a037091750c3f315, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74657be6232ad22aaca548d82491de41, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=5 2024-11-19T07:13:33,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf73eccbee4f01f5a86b87f17dcdb8b, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a637105eb1d8b75696b3cfabe5d8173, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df93dc30d9c55d765ada1f50e320789, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dba97405d3b35647d42d266df5e81cec, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d703151d82f3a429f348967988283542, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f38687f659a16d9e2a781f2a75d514a, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9357ae6c6e8337f25dacbf0a10701ef6, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08f6487d04261d6f0713b4b5adeacf2e, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=611b07fbb4fb5e3ea7255d9fa0f9fe40, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=126b77a274d2ba63b12dd23c9e7ababb, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3aa6780c71c61c778946d2086357143d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32d987cca115e6f476cdb5686be2b726, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,261 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T07:13:33,261 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732000473261 2024-11-19T07:13:33,261 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732000533261 2024-11-19T07:13:33,262 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-19T07:13:33,262 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T07:13:33,262 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,262 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T07:13:33,262 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T07:13:33,262 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T07:13:33,262 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T07:13:33,262 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T07:13:33,263 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T07:13:33,264 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T07:13:33,266 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T07:13:33,266 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,266 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:33,266 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,267 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,267 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,267 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,267 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,267 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:33,267 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,267 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:33,267 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T07:13:33,268 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T07:13:33,268 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, REOPEN/MOVE 2024-11-19T07:13:33,269 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true 2024-11-19T07:13:33,269 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,269 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,269 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T07:13:33,269 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,269 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,270 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,270 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:33,270 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:33,270 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:33,270 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN 2024-11-19T07:13:33,270 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,270 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,270 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,270 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,270 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,270 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,270 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,270 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,270 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:33,270 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,270 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:33,270 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,271 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:33,271 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,271 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN 2024-11-19T07:13:33,271 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-19T07:13:33,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=65 2024-11-19T07:13:33,272 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=db0afee3eab9,42435,1732000404708, table=hbase:meta, region=1588230740 2024-11-19T07:13:33,272 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,42435,1732000404708, state=OPEN 2024-11-19T07:13:33,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,280 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,280 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,280 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,280 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN to state=OFFLINE, location=null, table=test, region=08f6487d04261d6f0713b4b5adeacf2e to restore RIT 2024-11-19T07:13:33,281 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN to state=OFFLINE, location=null, table=test, region=3aa6780c71c61c778946d2086357143d to restore RIT 2024-11-19T07:13:33,281 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN to state=OFFLINE, location=null, table=test, region=2f38687f659a16d9e2a781f2a75d514a to restore RIT 2024-11-19T07:13:33,281 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e880c03f4c433baa6242dd65ef2d8486, ASSIGN to state=OFFLINE, location=null, table=test, region=e880c03f4c433baa6242dd65ef2d8486 to restore RIT 2024-11-19T07:13:33,281 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732000443281 2024-11-19T07:13:33,282 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T07:13:33,282 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,283 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T07:13:33,285 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T07:13:33,285 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:33,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,285 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02309bab61f6207d73b9f19985e1fdb9, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dcc969f79695b5f0d224b605268d2d, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d9d31e1a2abc752a2056ed0e3a3fcec, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ece759217c055d9a295034ce69dd86d, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e880c03f4c433baa6242dd65ef2d8486, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49b850b07eedb76caa94e4c93ac5c832, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db4ba820395e8b4a2b38d57f8a222e97, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66fe58970f9c4a1cba9c3e3da843ed3d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43676ec232db06c9bf4af64db1ede4f5, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4421c1189c5fdff3a40dcc58728c41ab, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbb518f4693490310eca052bf28f87fc, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82941673e9ad03e5a037091750c3f315, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74657be6232ad22aaca548d82491de41, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=5 2024-11-19T07:13:33,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf73eccbee4f01f5a86b87f17dcdb8b, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a637105eb1d8b75696b3cfabe5d8173, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df93dc30d9c55d765ada1f50e320789, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dba97405d3b35647d42d266df5e81cec, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d703151d82f3a429f348967988283542, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f38687f659a16d9e2a781f2a75d514a, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9357ae6c6e8337f25dacbf0a10701ef6, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08f6487d04261d6f0713b4b5adeacf2e, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=611b07fbb4fb5e3ea7255d9fa0f9fe40, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=126b77a274d2ba63b12dd23c9e7ababb, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3aa6780c71c61c778946d2086357143d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32d987cca115e6f476cdb5686be2b726, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,292 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T07:13:33,292 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732000473292 2024-11-19T07:13:33,293 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732000533293 2024-11-19T07:13:33,293 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-19T07:13:33,293 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T07:13:33,293 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,293 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T07:13:33,293 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T07:13:33,293 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T07:13:33,293 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T07:13:33,293 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T07:13:33,294 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T07:13:33,295 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T07:13:33,297 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T07:13:33,297 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,297 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:33,297 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,297 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,297 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,297 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,297 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:33,297 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,297 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:33,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T07:13:33,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T07:13:33,299 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, REOPEN/MOVE 2024-11-19T07:13:33,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true 2024-11-19T07:13:33,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T07:13:33,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:33,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:33,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:33,300 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,300 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,300 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,300 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,300 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,300 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,301 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,301 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,301 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:33,301 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,301 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:33,301 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,301 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN 2024-11-19T07:13:33,301 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-19T07:13:33,302 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=65 2024-11-19T07:13:33,302 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=db0afee3eab9,42435,1732000404708, table=hbase:meta, region=1588230740 2024-11-19T07:13:33,302 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,42435,1732000404708, state=OPEN 2024-11-19T07:13:33,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,311 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,311 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,311 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,312 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN to state=OFFLINE, location=null, table=test, region=08f6487d04261d6f0713b4b5adeacf2e to restore RIT 2024-11-19T07:13:33,312 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN to state=OFFLINE, location=null, table=test, region=3aa6780c71c61c778946d2086357143d to restore RIT 2024-11-19T07:13:33,312 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f38687f659a16d9e2a781f2a75d514a, ASSIGN to state=OFFLINE, location=null, table=test, region=2f38687f659a16d9e2a781f2a75d514a to restore RIT 2024-11-19T07:13:33,312 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732000443312 2024-11-19T07:13:33,313 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T07:13:33,313 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,314 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T07:13:33,315 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T07:13:33,315 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:33,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,315 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02309bab61f6207d73b9f19985e1fdb9, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dcc969f79695b5f0d224b605268d2d, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d9d31e1a2abc752a2056ed0e3a3fcec, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ece759217c055d9a295034ce69dd86d, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e880c03f4c433baa6242dd65ef2d8486, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49b850b07eedb76caa94e4c93ac5c832, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db4ba820395e8b4a2b38d57f8a222e97, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66fe58970f9c4a1cba9c3e3da843ed3d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43676ec232db06c9bf4af64db1ede4f5, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4421c1189c5fdff3a40dcc58728c41ab, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbb518f4693490310eca052bf28f87fc, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82941673e9ad03e5a037091750c3f315, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74657be6232ad22aaca548d82491de41, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=5 2024-11-19T07:13:33,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf73eccbee4f01f5a86b87f17dcdb8b, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a637105eb1d8b75696b3cfabe5d8173, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df93dc30d9c55d765ada1f50e320789, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dba97405d3b35647d42d266df5e81cec, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d703151d82f3a429f348967988283542, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f38687f659a16d9e2a781f2a75d514a, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9357ae6c6e8337f25dacbf0a10701ef6, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08f6487d04261d6f0713b4b5adeacf2e, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=611b07fbb4fb5e3ea7255d9fa0f9fe40, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=126b77a274d2ba63b12dd23c9e7ababb, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3aa6780c71c61c778946d2086357143d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32d987cca115e6f476cdb5686be2b726, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,322 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T07:13:33,322 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732000473322 2024-11-19T07:13:33,322 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732000533322 2024-11-19T07:13:33,322 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-19T07:13:33,322 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T07:13:33,323 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,323 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T07:13:33,323 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T07:13:33,323 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T07:13:33,323 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T07:13:33,323 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T07:13:33,323 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T07:13:33,324 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T07:13:33,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T07:13:33,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:33,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:33,328 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,328 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:33,327 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,328 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T07:13:33,328 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T07:13:33,329 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, REOPEN/MOVE 2024-11-19T07:13:33,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true 2024-11-19T07:13:33,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T07:13:33,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:33,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:33,330 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,330 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,330 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,330 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,330 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,330 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,330 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,330 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,330 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:33,330 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,330 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN 2024-11-19T07:13:33,330 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-19T07:13:33,331 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=65 2024-11-19T07:13:33,331 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=db0afee3eab9,42435,1732000404708, table=hbase:meta, region=1588230740 2024-11-19T07:13:33,331 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,42435,1732000404708, state=OPEN 2024-11-19T07:13:33,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,380 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,380 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,380 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,381 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN to state=OFFLINE, location=null, table=test, region=08f6487d04261d6f0713b4b5adeacf2e to restore RIT 2024-11-19T07:13:33,381 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3aa6780c71c61c778946d2086357143d, ASSIGN to state=OFFLINE, location=null, table=test, region=3aa6780c71c61c778946d2086357143d to restore RIT 2024-11-19T07:13:33,383 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732000443383 2024-11-19T07:13:33,383 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T07:13:33,383 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,384 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T07:13:33,385 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T07:13:33,386 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:33,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,386 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02309bab61f6207d73b9f19985e1fdb9, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dcc969f79695b5f0d224b605268d2d, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d9d31e1a2abc752a2056ed0e3a3fcec, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ece759217c055d9a295034ce69dd86d, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e880c03f4c433baa6242dd65ef2d8486, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49b850b07eedb76caa94e4c93ac5c832, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db4ba820395e8b4a2b38d57f8a222e97, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66fe58970f9c4a1cba9c3e3da843ed3d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43676ec232db06c9bf4af64db1ede4f5, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4421c1189c5fdff3a40dcc58728c41ab, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbb518f4693490310eca052bf28f87fc, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82941673e9ad03e5a037091750c3f315, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74657be6232ad22aaca548d82491de41, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=5 2024-11-19T07:13:33,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf73eccbee4f01f5a86b87f17dcdb8b, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a637105eb1d8b75696b3cfabe5d8173, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df93dc30d9c55d765ada1f50e320789, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dba97405d3b35647d42d266df5e81cec, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d703151d82f3a429f348967988283542, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f38687f659a16d9e2a781f2a75d514a, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9357ae6c6e8337f25dacbf0a10701ef6, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,391 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08f6487d04261d6f0713b4b5adeacf2e, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,391 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=611b07fbb4fb5e3ea7255d9fa0f9fe40, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,391 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=126b77a274d2ba63b12dd23c9e7ababb, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,391 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3aa6780c71c61c778946d2086357143d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,391 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32d987cca115e6f476cdb5686be2b726, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,391 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T07:13:33,391 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732000473391 2024-11-19T07:13:33,391 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732000533391 2024-11-19T07:13:33,391 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-19T07:13:33,391 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T07:13:33,391 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,391 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T07:13:33,391 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T07:13:33,391 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T07:13:33,391 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T07:13:33,392 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T07:13:33,392 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T07:13:33,393 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T07:13:33,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T07:13:33,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:33,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:33,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:33,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T07:13:33,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T07:13:33,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, REOPEN/MOVE 2024-11-19T07:13:33,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true 2024-11-19T07:13:33,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T07:13:33,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:33,397 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,397 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,397 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,397 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,397 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,397 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,397 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,397 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,397 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN 2024-11-19T07:13:33,397 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-19T07:13:33,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=65 2024-11-19T07:13:33,398 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=db0afee3eab9,42435,1732000404708, table=hbase:meta, region=1588230740 2024-11-19T07:13:33,398 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,42435,1732000404708, state=OPEN 2024-11-19T07:13:33,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,417 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,417 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,417 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,418 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=08f6487d04261d6f0713b4b5adeacf2e, ASSIGN to state=OFFLINE, location=null, table=test, region=08f6487d04261d6f0713b4b5adeacf2e to restore RIT 2024-11-19T07:13:33,418 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732000443418 2024-11-19T07:13:33,418 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T07:13:33,418 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,420 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T07:13:33,421 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T07:13:33,421 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,421 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02309bab61f6207d73b9f19985e1fdb9, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dcc969f79695b5f0d224b605268d2d, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d9d31e1a2abc752a2056ed0e3a3fcec, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ece759217c055d9a295034ce69dd86d, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e880c03f4c433baa6242dd65ef2d8486, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49b850b07eedb76caa94e4c93ac5c832, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db4ba820395e8b4a2b38d57f8a222e97, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66fe58970f9c4a1cba9c3e3da843ed3d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43676ec232db06c9bf4af64db1ede4f5, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4421c1189c5fdff3a40dcc58728c41ab, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbb518f4693490310eca052bf28f87fc, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82941673e9ad03e5a037091750c3f315, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74657be6232ad22aaca548d82491de41, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=5 2024-11-19T07:13:33,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf73eccbee4f01f5a86b87f17dcdb8b, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a637105eb1d8b75696b3cfabe5d8173, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df93dc30d9c55d765ada1f50e320789, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dba97405d3b35647d42d266df5e81cec, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,427 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d703151d82f3a429f348967988283542, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,427 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f38687f659a16d9e2a781f2a75d514a, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,427 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9357ae6c6e8337f25dacbf0a10701ef6, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,427 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08f6487d04261d6f0713b4b5adeacf2e, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,427 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=611b07fbb4fb5e3ea7255d9fa0f9fe40, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,427 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=126b77a274d2ba63b12dd23c9e7ababb, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,427 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3aa6780c71c61c778946d2086357143d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,427 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32d987cca115e6f476cdb5686be2b726, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,427 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T07:13:33,427 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732000473427 2024-11-19T07:13:33,427 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732000533427 2024-11-19T07:13:33,428 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-19T07:13:33,428 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T07:13:33,428 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,428 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T07:13:33,428 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T07:13:33,428 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T07:13:33,428 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T07:13:33,428 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T07:13:33,429 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T07:13:33,430 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T07:13:33,431 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T07:13:33,431 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,431 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,431 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:33,431 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,431 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:33,431 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,432 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:33,432 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T07:13:33,432 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T07:13:33,433 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, REOPEN/MOVE 2024-11-19T07:13:33,433 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true 2024-11-19T07:13:33,433 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,433 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,433 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T07:13:33,433 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,433 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,433 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,433 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,433 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,433 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,434 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,434 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,434 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,434 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,434 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-19T07:13:33,434 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=65 2024-11-19T07:13:33,434 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=db0afee3eab9,42435,1732000404708, table=hbase:meta, region=1588230740 2024-11-19T07:13:33,435 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,42435,1732000404708, state=OPEN 2024-11-19T07:13:33,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,447 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,447 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,447 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,449 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732000443449 2024-11-19T07:13:33,449 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T07:13:33,449 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,451 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T07:13:33,453 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T07:13:33,453 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,453 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,455 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02309bab61f6207d73b9f19985e1fdb9, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dcc969f79695b5f0d224b605268d2d, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d9d31e1a2abc752a2056ed0e3a3fcec, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ece759217c055d9a295034ce69dd86d, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e880c03f4c433baa6242dd65ef2d8486, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49b850b07eedb76caa94e4c93ac5c832, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db4ba820395e8b4a2b38d57f8a222e97, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,457 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66fe58970f9c4a1cba9c3e3da843ed3d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,457 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43676ec232db06c9bf4af64db1ede4f5, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,457 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4421c1189c5fdff3a40dcc58728c41ab, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,457 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbb518f4693490310eca052bf28f87fc, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,457 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82941673e9ad03e5a037091750c3f315, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,457 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,457 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74657be6232ad22aaca548d82491de41, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=5 2024-11-19T07:13:33,457 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf73eccbee4f01f5a86b87f17dcdb8b, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a637105eb1d8b75696b3cfabe5d8173, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df93dc30d9c55d765ada1f50e320789, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dba97405d3b35647d42d266df5e81cec, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d703151d82f3a429f348967988283542, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f38687f659a16d9e2a781f2a75d514a, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9357ae6c6e8337f25dacbf0a10701ef6, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08f6487d04261d6f0713b4b5adeacf2e, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=611b07fbb4fb5e3ea7255d9fa0f9fe40, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=126b77a274d2ba63b12dd23c9e7ababb, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3aa6780c71c61c778946d2086357143d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32d987cca115e6f476cdb5686be2b726, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,459 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T07:13:33,459 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732000473459 2024-11-19T07:13:33,459 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732000533459 2024-11-19T07:13:33,459 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-19T07:13:33,459 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T07:13:33,459 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,459 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T07:13:33,459 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T07:13:33,460 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T07:13:33,460 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T07:13:33,460 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T07:13:33,460 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T07:13:33,461 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T07:13:33,463 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T07:13:33,463 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,463 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,463 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:33,463 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,463 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:33,463 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T07:13:33,464 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T07:13:33,464 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, REOPEN/MOVE 2024-11-19T07:13:33,465 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true 2024-11-19T07:13:33,465 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,465 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,465 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T07:13:33,465 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,466 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,466 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,466 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,466 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,466 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,466 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,466 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.1732000406457, worker=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,466 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-19T07:13:33,467 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=65 2024-11-19T07:13:33,467 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=db0afee3eab9,42435,1732000404708, table=hbase:meta, region=1588230740 2024-11-19T07:13:33,467 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,42435,1732000404708, state=OPEN 2024-11-19T07:13:33,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,480 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,480 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,480 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,482 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732000443482 2024-11-19T07:13:33,482 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T07:13:33,482 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,483 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T07:13:33,484 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T07:13:33,484 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,485 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02309bab61f6207d73b9f19985e1fdb9, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dcc969f79695b5f0d224b605268d2d, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d9d31e1a2abc752a2056ed0e3a3fcec, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ece759217c055d9a295034ce69dd86d, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e880c03f4c433baa6242dd65ef2d8486, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49b850b07eedb76caa94e4c93ac5c832, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db4ba820395e8b4a2b38d57f8a222e97, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66fe58970f9c4a1cba9c3e3da843ed3d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43676ec232db06c9bf4af64db1ede4f5, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4421c1189c5fdff3a40dcc58728c41ab, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbb518f4693490310eca052bf28f87fc, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82941673e9ad03e5a037091750c3f315, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74657be6232ad22aaca548d82491de41, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=5 2024-11-19T07:13:33,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf73eccbee4f01f5a86b87f17dcdb8b, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a637105eb1d8b75696b3cfabe5d8173, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df93dc30d9c55d765ada1f50e320789, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dba97405d3b35647d42d266df5e81cec, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d703151d82f3a429f348967988283542, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f38687f659a16d9e2a781f2a75d514a, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9357ae6c6e8337f25dacbf0a10701ef6, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08f6487d04261d6f0713b4b5adeacf2e, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=611b07fbb4fb5e3ea7255d9fa0f9fe40, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=126b77a274d2ba63b12dd23c9e7ababb, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3aa6780c71c61c778946d2086357143d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32d987cca115e6f476cdb5686be2b726, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,492 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T07:13:33,492 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732000473492 2024-11-19T07:13:33,492 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732000533492 2024-11-19T07:13:33,492 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-19T07:13:33,492 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T07:13:33,492 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,492 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T07:13:33,492 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T07:13:33,493 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T07:13:33,493 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T07:13:33,493 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T07:13:33,493 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T07:13:33,494 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T07:13:33,496 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T07:13:33,496 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,496 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:33,496 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,496 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T07:13:33,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T07:13:33,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T07:13:33,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, REOPEN/MOVE 2024-11-19T07:13:33,498 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true 2024-11-19T07:13:33,498 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,498 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,499 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T07:13:33,499 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,499 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,499 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,499 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,499 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,499 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42435,1732000404708 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,499 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-19T07:13:33,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=65 2024-11-19T07:13:33,500 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=db0afee3eab9,42435,1732000404708, table=hbase:meta, region=1588230740 2024-11-19T07:13:33,500 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,42435,1732000404708, state=OPEN 2024-11-19T07:13:33,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,512 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,512 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,512 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,513 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732000443513 2024-11-19T07:13:33,514 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T07:13:33,513 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,515 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T07:13:33,517 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T07:13:33,517 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T07:13:33,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,517 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02309bab61f6207d73b9f19985e1fdb9, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dcc969f79695b5f0d224b605268d2d, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d9d31e1a2abc752a2056ed0e3a3fcec, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ece759217c055d9a295034ce69dd86d, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e880c03f4c433baa6242dd65ef2d8486, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49b850b07eedb76caa94e4c93ac5c832, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db4ba820395e8b4a2b38d57f8a222e97, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66fe58970f9c4a1cba9c3e3da843ed3d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43676ec232db06c9bf4af64db1ede4f5, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4421c1189c5fdff3a40dcc58728c41ab, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbb518f4693490310eca052bf28f87fc, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82941673e9ad03e5a037091750c3f315, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74657be6232ad22aaca548d82491de41, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=5 2024-11-19T07:13:33,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf73eccbee4f01f5a86b87f17dcdb8b, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a637105eb1d8b75696b3cfabe5d8173, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df93dc30d9c55d765ada1f50e320789, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dba97405d3b35647d42d266df5e81cec, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d703151d82f3a429f348967988283542, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f38687f659a16d9e2a781f2a75d514a, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9357ae6c6e8337f25dacbf0a10701ef6, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08f6487d04261d6f0713b4b5adeacf2e, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=611b07fbb4fb5e3ea7255d9fa0f9fe40, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=126b77a274d2ba63b12dd23c9e7ababb, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3aa6780c71c61c778946d2086357143d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32d987cca115e6f476cdb5686be2b726, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,523 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T07:13:33,523 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732000473523 2024-11-19T07:13:33,523 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732000533523 2024-11-19T07:13:33,524 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-19T07:13:33,524 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T07:13:33,524 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,524 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T07:13:33,524 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T07:13:33,524 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T07:13:33,524 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T07:13:33,524 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T07:13:33,525 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T07:13:33,525 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,525 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T07:13:33,527 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T07:13:33,527 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,527 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:33,527 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T07:13:33,527 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T07:13:33,527 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T07:13:33,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, REOPEN/MOVE 2024-11-19T07:13:33,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true 2024-11-19T07:13:33,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T07:13:33,528 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,528 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,528 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,528 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,528 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-19T07:13:33,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=65 2024-11-19T07:13:33,529 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=db0afee3eab9,42435,1732000404708, table=hbase:meta, region=1588230740 2024-11-19T07:13:33,529 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,42435,1732000404708, state=OPEN 2024-11-19T07:13:33,535 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,542 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,542 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,542 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,543 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732000443543 2024-11-19T07:13:33,543 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T07:13:33,543 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,544 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T07:13:33,545 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T07:13:33,545 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,546 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,548 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02309bab61f6207d73b9f19985e1fdb9, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dcc969f79695b5f0d224b605268d2d, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d9d31e1a2abc752a2056ed0e3a3fcec, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ece759217c055d9a295034ce69dd86d, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e880c03f4c433baa6242dd65ef2d8486, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49b850b07eedb76caa94e4c93ac5c832, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db4ba820395e8b4a2b38d57f8a222e97, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66fe58970f9c4a1cba9c3e3da843ed3d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43676ec232db06c9bf4af64db1ede4f5, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4421c1189c5fdff3a40dcc58728c41ab, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbb518f4693490310eca052bf28f87fc, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82941673e9ad03e5a037091750c3f315, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74657be6232ad22aaca548d82491de41, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=5 2024-11-19T07:13:33,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf73eccbee4f01f5a86b87f17dcdb8b, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a637105eb1d8b75696b3cfabe5d8173, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df93dc30d9c55d765ada1f50e320789, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dba97405d3b35647d42d266df5e81cec, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d703151d82f3a429f348967988283542, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f38687f659a16d9e2a781f2a75d514a, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9357ae6c6e8337f25dacbf0a10701ef6, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08f6487d04261d6f0713b4b5adeacf2e, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=611b07fbb4fb5e3ea7255d9fa0f9fe40, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=126b77a274d2ba63b12dd23c9e7ababb, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3aa6780c71c61c778946d2086357143d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32d987cca115e6f476cdb5686be2b726, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,550 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T07:13:33,550 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732000473550 2024-11-19T07:13:33,551 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732000533550 2024-11-19T07:13:33,551 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-19T07:13:33,551 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T07:13:33,551 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,551 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T07:13:33,551 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T07:13:33,551 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T07:13:33,551 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T07:13:33,551 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T07:13:33,551 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T07:13:33,552 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T07:13:33,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T07:13:33,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T07:13:33,554 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T07:13:33,554 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T07:13:33,555 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, REOPEN/MOVE 2024-11-19T07:13:33,555 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true 2024-11-19T07:13:33,555 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,555 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,555 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,555 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,555 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,555 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-19T07:13:33,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=65 2024-11-19T07:13:33,556 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=db0afee3eab9,42435,1732000404708, table=hbase:meta, region=1588230740 2024-11-19T07:13:33,556 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,42435,1732000404708, state=OPEN 2024-11-19T07:13:33,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,564 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,564 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,564 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,565 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732000443565 2024-11-19T07:13:33,565 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T07:13:33,565 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,566 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T07:13:33,567 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T07:13:33,568 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,568 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,568 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,569 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02309bab61f6207d73b9f19985e1fdb9, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,569 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dcc969f79695b5f0d224b605268d2d, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,570 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d9d31e1a2abc752a2056ed0e3a3fcec, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,570 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ece759217c055d9a295034ce69dd86d, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,570 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e880c03f4c433baa6242dd65ef2d8486, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,570 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49b850b07eedb76caa94e4c93ac5c832, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,570 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db4ba820395e8b4a2b38d57f8a222e97, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,570 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66fe58970f9c4a1cba9c3e3da843ed3d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,570 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43676ec232db06c9bf4af64db1ede4f5, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,570 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4421c1189c5fdff3a40dcc58728c41ab, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbb518f4693490310eca052bf28f87fc, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82941673e9ad03e5a037091750c3f315, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74657be6232ad22aaca548d82491de41, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=5 2024-11-19T07:13:33,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf73eccbee4f01f5a86b87f17dcdb8b, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a637105eb1d8b75696b3cfabe5d8173, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df93dc30d9c55d765ada1f50e320789, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dba97405d3b35647d42d266df5e81cec, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d703151d82f3a429f348967988283542, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f38687f659a16d9e2a781f2a75d514a, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9357ae6c6e8337f25dacbf0a10701ef6, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08f6487d04261d6f0713b4b5adeacf2e, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=611b07fbb4fb5e3ea7255d9fa0f9fe40, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=126b77a274d2ba63b12dd23c9e7ababb, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3aa6780c71c61c778946d2086357143d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32d987cca115e6f476cdb5686be2b726, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,572 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T07:13:33,572 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732000473572 2024-11-19T07:13:33,572 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732000533572 2024-11-19T07:13:33,572 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-19T07:13:33,573 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T07:13:33,573 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,573 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T07:13:33,573 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T07:13:33,573 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T07:13:33,573 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T07:13:33,573 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T07:13:33,574 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T07:13:33,575 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T07:13:33,576 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T07:13:33,576 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T07:13:33,576 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T07:13:33,577 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T07:13:33,577 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, REOPEN/MOVE 2024-11-19T07:13:33,578 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true 2024-11-19T07:13:33,578 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,578 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,578 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure db0afee3eab9%2C37853%2C1732000404756.meta.1732000406864.meta, worker=db0afee3eab9,34425,1732000404604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T07:13:33,578 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-19T07:13:33,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=65 2024-11-19T07:13:33,579 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=db0afee3eab9,42435,1732000404708, table=hbase:meta, region=1588230740 2024-11-19T07:13:33,579 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,42435,1732000404708, state=OPEN 2024-11-19T07:13:33,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,585 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,585 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,585 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,586 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732000443586 2024-11-19T07:13:33,586 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T07:13:33,586 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,588 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T07:13:33,591 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T07:13:33,591 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true 2024-11-19T07:13:33,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,591 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02309bab61f6207d73b9f19985e1fdb9, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dcc969f79695b5f0d224b605268d2d, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d9d31e1a2abc752a2056ed0e3a3fcec, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ece759217c055d9a295034ce69dd86d, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e880c03f4c433baa6242dd65ef2d8486, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49b850b07eedb76caa94e4c93ac5c832, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db4ba820395e8b4a2b38d57f8a222e97, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66fe58970f9c4a1cba9c3e3da843ed3d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43676ec232db06c9bf4af64db1ede4f5, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4421c1189c5fdff3a40dcc58728c41ab, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbb518f4693490310eca052bf28f87fc, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82941673e9ad03e5a037091750c3f315, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74657be6232ad22aaca548d82491de41, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=5 2024-11-19T07:13:33,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf73eccbee4f01f5a86b87f17dcdb8b, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a637105eb1d8b75696b3cfabe5d8173, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df93dc30d9c55d765ada1f50e320789, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dba97405d3b35647d42d266df5e81cec, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d703151d82f3a429f348967988283542, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f38687f659a16d9e2a781f2a75d514a, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9357ae6c6e8337f25dacbf0a10701ef6, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08f6487d04261d6f0713b4b5adeacf2e, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=611b07fbb4fb5e3ea7255d9fa0f9fe40, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=126b77a274d2ba63b12dd23c9e7ababb, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3aa6780c71c61c778946d2086357143d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32d987cca115e6f476cdb5686be2b726, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,595 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T07:13:33,595 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732000473595 2024-11-19T07:13:33,595 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732000533595 2024-11-19T07:13:33,595 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-19T07:13:33,595 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T07:13:33,595 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,595 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T07:13:33,595 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T07:13:33,595 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T07:13:33,595 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T07:13:33,595 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T07:13:33,596 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T07:13:33,596 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T07:13:33,598 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T07:13:33,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T07:13:33,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T07:13:33,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=74657be6232ad22aaca548d82491de41, REOPEN/MOVE 2024-11-19T07:13:33,599 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true 2024-11-19T07:13:33,599 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T07:13:33,599 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-19T07:13:33,600 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=65 2024-11-19T07:13:33,600 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=db0afee3eab9,42435,1732000404708, table=hbase:meta, region=1588230740 2024-11-19T07:13:33,600 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,42435,1732000404708, state=OPEN 2024-11-19T07:13:33,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T07:13:33,606 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,606 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,606 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T07:13:33,607 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732000443607 2024-11-19T07:13:33,608 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T07:13:33,608 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T07:13:33,609 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T07:13:33,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=60, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2f38687f659a16d9e2a781f2a75d514a, server=db0afee3eab9,34425,1732000404604:java.lang.RuntimeException: inject code bug; ServerCrashProcedure db0afee3eab9,37853,1732000404756, splitWal=true, meta=true exec-time=2.7380 sec 2024-11-19T07:13:33,612 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02309bab61f6207d73b9f19985e1fdb9, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,612 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dcc969f79695b5f0d224b605268d2d, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,613 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d9d31e1a2abc752a2056ed0e3a3fcec, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,613 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ece759217c055d9a295034ce69dd86d, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,613 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e880c03f4c433baa6242dd65ef2d8486, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,613 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49b850b07eedb76caa94e4c93ac5c832, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,613 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db4ba820395e8b4a2b38d57f8a222e97, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,613 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66fe58970f9c4a1cba9c3e3da843ed3d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,613 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43676ec232db06c9bf4af64db1ede4f5, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,613 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4421c1189c5fdff3a40dcc58728c41ab, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,613 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbb518f4693490310eca052bf28f87fc, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,613 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82941673e9ad03e5a037091750c3f315, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,613 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ef4078aa4f5f4b6f4dc7188dd2f1cb, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,613 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74657be6232ad22aaca548d82491de41, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=5 2024-11-19T07:13:33,613 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf73eccbee4f01f5a86b87f17dcdb8b, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,613 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a637105eb1d8b75696b3cfabe5d8173, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,614 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df93dc30d9c55d765ada1f50e320789, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,614 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dba97405d3b35647d42d266df5e81cec, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,614 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d703151d82f3a429f348967988283542, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,614 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f38687f659a16d9e2a781f2a75d514a, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,614 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9357ae6c6e8337f25dacbf0a10701ef6, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,614 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08f6487d04261d6f0713b4b5adeacf2e, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,614 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=611b07fbb4fb5e3ea7255d9fa0f9fe40, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,614 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=126b77a274d2ba63b12dd23c9e7ababb, regionState=OPEN, lastHost=db0afee3eab9,34425,1732000404604, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,614 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3aa6780c71c61c778946d2086357143d, regionState=OPENING, lastHost=db0afee3eab9,37853,1732000404756, regionLocation=db0afee3eab9,34425,1732000404604, openSeqNum=2 2024-11-19T07:13:33,614 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32d987cca115e6f476cdb5686be2b726, regionState=OPEN, lastHost=db0afee3eab9,42435,1732000404708, regionLocation=db0afee3eab9,42435,1732000404708, openSeqNum=2 2024-11-19T07:13:33,614 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T07:13:33,614 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732000473614 2024-11-19T07:13:33,614 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732000533614 2024-11-19T07:13:33,614 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-19T07:13:33,615 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] assignment.AssignmentManager(1308): No matching procedure found for db0afee3eab9,34425,1732000404604 transition on state=OPENING, location=db0afee3eab9,34425,1732000404604, table=test, region=bbb518f4693490310eca052bf28f87fc to OPENED 2024-11-19T07:13:33,616 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] assignment.AssignmentManager(1308): No matching procedure found for db0afee3eab9,42435,1732000404708 transition on state=OPENING, location=db0afee3eab9,42435,1732000404708, table=test, region=08f6487d04261d6f0713b4b5adeacf2e to OPENED 2024-11-19T07:13:33,617 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2353): TRANSITION REPORTED server { host_name: "db0afee3eab9" port: 42435 start_code: 1732000404708 } transition { transition_code: OPENED region_info { region_id: 1732000407304 table_name { namespace: "default" qualifier: "test" } start_key: "vvv" end_key: "www" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 76 initiating_master_active_time: 1732000404850 } 2024-11-19T07:13:33,617 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2353): TRANSITION REPORTED server { host_name: "db0afee3eab9" port: 34425 start_code: 1732000404604 } transition { transition_code: OPENED region_info { region_id: 1732000407304 table_name { namespace: "default" qualifier: "test" } start_key: "kkk" end_key: "lll" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 82 initiating_master_active_time: 1732000404850 } 2024-11-19T07:13:33,617 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1732000407304.08f6487d04261d6f0713b4b5adeacf2e. 2024-11-19T07:13:33,617 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1732000407304.bbb518f4693490310eca052bf28f87fc. 2024-11-19T07:13:33,618 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(153): Opened test,kkk,1732000407304.bbb518f4693490310eca052bf28f87fc. 2024-11-19T07:13:33,618 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,vvv,1732000407304.08f6487d04261d6f0713b4b5adeacf2e. 2024-11-19T07:13:33,618 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,hhh,1732000407304.66fe58970f9c4a1cba9c3e3da843ed3d. 2024-11-19T07:13:33,618 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 66fe58970f9c4a1cba9c3e3da843ed3d, NAME => 'test,hhh,1732000407304.66fe58970f9c4a1cba9c3e3da843ed3d.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-19T07:13:33,618 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 66fe58970f9c4a1cba9c3e3da843ed3d 2024-11-19T07:13:33,618 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,hhh,1732000407304.66fe58970f9c4a1cba9c3e3da843ed3d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:33,618 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 66fe58970f9c4a1cba9c3e3da843ed3d 2024-11-19T07:13:33,618 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 66fe58970f9c4a1cba9c3e3da843ed3d 2024-11-19T07:13:33,623 INFO [StoreOpener-66fe58970f9c4a1cba9c3e3da843ed3d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 66fe58970f9c4a1cba9c3e3da843ed3d 2024-11-19T07:13:33,624 INFO [StoreOpener-66fe58970f9c4a1cba9c3e3da843ed3d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66fe58970f9c4a1cba9c3e3da843ed3d columnFamilyName family 2024-11-19T07:13:33,625 DEBUG [StoreOpener-66fe58970f9c4a1cba9c3e3da843ed3d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:33,625 INFO [StoreOpener-66fe58970f9c4a1cba9c3e3da843ed3d-1 {}] regionserver.HStore(327): Store=66fe58970f9c4a1cba9c3e3da843ed3d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:33,626 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 66fe58970f9c4a1cba9c3e3da843ed3d 2024-11-19T07:13:33,626 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/66fe58970f9c4a1cba9c3e3da843ed3d 2024-11-19T07:13:33,629 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/66fe58970f9c4a1cba9c3e3da843ed3d 2024-11-19T07:13:33,630 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 66fe58970f9c4a1cba9c3e3da843ed3d 2024-11-19T07:13:33,630 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 66fe58970f9c4a1cba9c3e3da843ed3d 2024-11-19T07:13:33,632 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 66fe58970f9c4a1cba9c3e3da843ed3d 2024-11-19T07:13:33,633 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 66fe58970f9c4a1cba9c3e3da843ed3d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72056046, jitterRate=0.07371875643730164}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:33,634 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 66fe58970f9c4a1cba9c3e3da843ed3d 2024-11-19T07:13:33,634 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 66fe58970f9c4a1cba9c3e3da843ed3d: Running coprocessor pre-open hook at 1732000413619Writing region info on filesystem at 1732000413619Initializing all the Stores at 1732000413620 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000413620Cleaning up temporary data from old regions at 1732000413630 (+10 ms)Running coprocessor post-open hooks at 1732000413634 (+4 ms)Region opened successfully at 1732000413634 2024-11-19T07:13:33,634 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=352 (was 386), OpenFileDescriptor=624 (was 647), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=357 (was 336) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=12205 (was 12301) 2024-11-19T07:13:33,634 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-19T07:13:33,634 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T07:13:33,635 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T07:13:33,635 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T07:13:33,635 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1732000407304.66fe58970f9c4a1cba9c3e3da843ed3d., pid=77, masterSystemTime=1732000412504 2024-11-19T07:13:33,636 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T07:13:33,636 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T07:13:33,636 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-19T07:13:33,636 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1846701803, stopped=false 2024-11-19T07:13:33,636 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43779 {}] assignment.AssignmentManager(1308): No matching procedure found for db0afee3eab9,42435,1732000404708 transition on state=OPENING, location=db0afee3eab9,42435,1732000404708, table=test, region=66fe58970f9c4a1cba9c3e3da843ed3d to OPENED 2024-11-19T07:13:33,637 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=db0afee3eab9,43779,1732000403834 2024-11-19T07:13:33,637 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1732000407304.66fe58970f9c4a1cba9c3e3da843ed3d. 2024-11-19T07:13:33,637 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,hhh,1732000407304.66fe58970f9c4a1cba9c3e3da843ed3d. 2024-11-19T07:13:33,637 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open test,qqq,1732000407304.2df93dc30d9c55d765ada1f50e320789. 2024-11-19T07:13:33,637 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => 2df93dc30d9c55d765ada1f50e320789, NAME => 'test,qqq,1732000407304.2df93dc30d9c55d765ada1f50e320789.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-19T07:13:33,638 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2df93dc30d9c55d765ada1f50e320789 2024-11-19T07:13:33,638 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated test,qqq,1732000407304.2df93dc30d9c55d765ada1f50e320789.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T07:13:33,638 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for 2df93dc30d9c55d765ada1f50e320789 2024-11-19T07:13:33,638 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for 2df93dc30d9c55d765ada1f50e320789 2024-11-19T07:13:33,641 INFO [StoreOpener-2df93dc30d9c55d765ada1f50e320789-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2df93dc30d9c55d765ada1f50e320789 2024-11-19T07:13:33,642 INFO [StoreOpener-2df93dc30d9c55d765ada1f50e320789-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2df93dc30d9c55d765ada1f50e320789 columnFamilyName family 2024-11-19T07:13:33,642 DEBUG [StoreOpener-2df93dc30d9c55d765ada1f50e320789-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T07:13:33,643 INFO [StoreOpener-2df93dc30d9c55d765ada1f50e320789-1 {}] regionserver.HStore(327): Store=2df93dc30d9c55d765ada1f50e320789/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T07:13:33,643 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for 2df93dc30d9c55d765ada1f50e320789 2024-11-19T07:13:33,644 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/2df93dc30d9c55d765ada1f50e320789 2024-11-19T07:13:33,646 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/2df93dc30d9c55d765ada1f50e320789 2024-11-19T07:13:33,646 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for 2df93dc30d9c55d765ada1f50e320789 2024-11-19T07:13:33,646 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for 2df93dc30d9c55d765ada1f50e320789 2024-11-19T07:13:33,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T07:13:33,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T07:13:33,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T07:13:33,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T07:13:33,648 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T07:13:33,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T07:13:33,649 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T07:13:33,649 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T07:13:33,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T07:13:33,649 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T07:13:33,649 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'db0afee3eab9,34425,1732000404604' ***** 2024-11-19T07:13:33,649 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T07:13:33,649 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T07:13:33,650 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T07:13:33,650 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T07:13:33,650 INFO [RS:0;db0afee3eab9:34425 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T07:13:33,650 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T07:13:33,650 INFO [RS:0;db0afee3eab9:34425 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T07:13:33,650 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'db0afee3eab9,42435,1732000404708' ***** 2024-11-19T07:13:33,650 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T07:13:33,650 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.HRegionServer(3091): Received CLOSE for 74657be6232ad22aaca548d82491de41 2024-11-19T07:13:33,650 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T07:13:33,650 INFO [RS:1;db0afee3eab9:42435 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T07:13:33,650 INFO [RS:1;db0afee3eab9:42435 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T07:13:33,650 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T07:13:33,650 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.HRegionServer(3091): Received CLOSE for 02309bab61f6207d73b9f19985e1fdb9 2024-11-19T07:13:33,651 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.HRegionServer(3091): Received CLOSE for bbb518f4693490310eca052bf28f87fc 2024-11-19T07:13:33,651 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.HRegionServer(3091): Received CLOSE for e880c03f4c433baa6242dd65ef2d8486 2024-11-19T07:13:33,651 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.HRegionServer(3091): Received CLOSE for dba97405d3b35647d42d266df5e81cec 2024-11-19T07:13:33,651 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.HRegionServer(3091): Received CLOSE for 9357ae6c6e8337f25dacbf0a10701ef6 2024-11-19T07:13:33,651 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.HRegionServer(3091): Received CLOSE for 12dcc969f79695b5f0d224b605268d2d 2024-11-19T07:13:33,651 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for 2df93dc30d9c55d765ada1f50e320789 2024-11-19T07:13:33,651 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.HRegionServer(3091): Received CLOSE for fdf73eccbee4f01f5a86b87f17dcdb8b 2024-11-19T07:13:33,651 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.HRegionServer(3091): Received CLOSE for 2f38687f659a16d9e2a781f2a75d514a 2024-11-19T07:13:33,651 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.HRegionServer(3091): Received CLOSE for 126b77a274d2ba63b12dd23c9e7ababb 2024-11-19T07:13:33,651 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.HRegionServer(3091): Received CLOSE for 49b850b07eedb76caa94e4c93ac5c832 2024-11-19T07:13:33,651 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.HRegionServer(3091): Received CLOSE for 82941673e9ad03e5a037091750c3f315 2024-11-19T07:13:33,651 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.HRegionServer(3091): Received CLOSE for 43676ec232db06c9bf4af64db1ede4f5 2024-11-19T07:13:33,651 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.HRegionServer(3091): Received CLOSE for 3aa6780c71c61c778946d2086357143d 2024-11-19T07:13:33,651 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.HRegionServer(959): stopping server db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,651 INFO [RS:0;db0afee3eab9:34425 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T07:13:33,652 INFO [RS:0;db0afee3eab9:34425 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;db0afee3eab9:34425. 2024-11-19T07:13:33,652 DEBUG [RS:0;db0afee3eab9:34425 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T07:13:33,652 DEBUG [RS:0;db0afee3eab9:34425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T07:13:33,652 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-19T07:13:33,652 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened 2df93dc30d9c55d765ada1f50e320789; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73096648, jitterRate=0.0892249345779419}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T07:13:33,652 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2df93dc30d9c55d765ada1f50e320789 2024-11-19T07:13:33,652 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for 2df93dc30d9c55d765ada1f50e320789: Running coprocessor pre-open hook at 1732000413638Writing region info on filesystem at 1732000413638Initializing all the Stores at 1732000413640 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732000413640Cleaning up temporary data from old regions at 1732000413646 (+6 ms)Running coprocessor post-open hooks at 1732000413652 (+6 ms)Region opened successfully at 1732000413652 2024-11-19T07:13:33,652 DEBUG [RS:0;db0afee3eab9:34425 {}] regionserver.HRegionServer(1351): Waiting on 126b77a274d2ba63b12dd23c9e7ababb, 12dcc969f79695b5f0d224b605268d2d, 2f38687f659a16d9e2a781f2a75d514a, 3aa6780c71c61c778946d2086357143d, 43676ec232db06c9bf4af64db1ede4f5, 49b850b07eedb76caa94e4c93ac5c832, 74657be6232ad22aaca548d82491de41, 82941673e9ad03e5a037091750c3f315, 9357ae6c6e8337f25dacbf0a10701ef6, bbb518f4693490310eca052bf28f87fc, dba97405d3b35647d42d266df5e81cec, e880c03f4c433baa6242dd65ef2d8486, fdf73eccbee4f01f5a86b87f17dcdb8b 2024-11-19T07:13:33,653 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.HRegionServer(3091): Received CLOSE for 7d9d31e1a2abc752a2056ed0e3a3fcec 2024-11-19T07:13:33,653 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.HRegionServer(3091): Received CLOSE for 66fe58970f9c4a1cba9c3e3da843ed3d 2024-11-19T07:13:33,653 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.HRegionServer(3091): Received CLOSE for 611b07fbb4fb5e3ea7255d9fa0f9fe40 2024-11-19T07:13:33,653 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.HRegionServer(3091): Received CLOSE for db4ba820395e8b4a2b38d57f8a222e97 2024-11-19T07:13:33,653 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.HRegionServer(3091): Received CLOSE for d703151d82f3a429f348967988283542 2024-11-19T07:13:33,653 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.HRegionServer(3091): Received CLOSE for 32d987cca115e6f476cdb5686be2b726 2024-11-19T07:13:33,653 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.HRegionServer(3091): Received CLOSE for 08f6487d04261d6f0713b4b5adeacf2e 2024-11-19T07:13:33,653 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.HRegionServer(3091): Received CLOSE for d2ef4078aa4f5f4b6f4dc7188dd2f1cb 2024-11-19T07:13:33,653 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.HRegionServer(3091): Received CLOSE for 3a637105eb1d8b75696b3cfabe5d8173 2024-11-19T07:13:33,653 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.HRegionServer(3091): Received CLOSE for 4421c1189c5fdff3a40dcc58728c41ab 2024-11-19T07:13:33,653 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.HRegionServer(3091): Received CLOSE for 1ece759217c055d9a295034ce69dd86d 2024-11-19T07:13:33,653 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.HRegionServer(959): stopping server db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,653 INFO [RS:1;db0afee3eab9:42435 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T07:13:33,653 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 02309bab61f6207d73b9f19985e1fdb9, disabling compactions & flushes 2024-11-19T07:13:33,654 INFO [RS:1;db0afee3eab9:42435 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;db0afee3eab9:42435. 2024-11-19T07:13:33,653 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 74657be6232ad22aaca548d82491de41, disabling compactions & flushes 2024-11-19T07:13:33,654 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1732000407304.02309bab61f6207d73b9f19985e1fdb9. 2024-11-19T07:13:33,654 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1732000407304.74657be6232ad22aaca548d82491de41. 2024-11-19T07:13:33,654 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732000407304.02309bab61f6207d73b9f19985e1fdb9. 2024-11-19T07:13:33,654 DEBUG [RS:1;db0afee3eab9:42435 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T07:13:33,654 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732000407304.74657be6232ad22aaca548d82491de41. 2024-11-19T07:13:33,654 DEBUG [RS:1;db0afee3eab9:42435 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T07:13:33,654 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1732000407304.02309bab61f6207d73b9f19985e1fdb9. after waiting 0 ms 2024-11-19T07:13:33,654 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732000407304.74657be6232ad22aaca548d82491de41. after waiting 0 ms 2024-11-19T07:13:33,654 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1732000407304.02309bab61f6207d73b9f19985e1fdb9. 2024-11-19T07:13:33,654 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732000407304.74657be6232ad22aaca548d82491de41. 2024-11-19T07:13:33,654 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T07:13:33,654 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T07:13:33,654 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T07:13:33,654 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-19T07:13:33,654 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-19T07:13:33,655 DEBUG [RS:1;db0afee3eab9:42435 {}] regionserver.HRegionServer(1351): Waiting on 02309bab61f6207d73b9f19985e1fdb9, 08f6487d04261d6f0713b4b5adeacf2e, 1588230740, 1ece759217c055d9a295034ce69dd86d, 2df93dc30d9c55d765ada1f50e320789, 32d987cca115e6f476cdb5686be2b726, 3a637105eb1d8b75696b3cfabe5d8173, 4421c1189c5fdff3a40dcc58728c41ab, 611b07fbb4fb5e3ea7255d9fa0f9fe40, 66fe58970f9c4a1cba9c3e3da843ed3d, 7d9d31e1a2abc752a2056ed0e3a3fcec, d2ef4078aa4f5f4b6f4dc7188dd2f1cb, d703151d82f3a429f348967988283542, db4ba820395e8b4a2b38d57f8a222e97 2024-11-19T07:13:33,655 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T07:13:33,655 INFO [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T07:13:33,655 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T07:13:33,655 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T07:13:33,655 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T07:13:33,655 INFO [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.66 KB heapSize=5.44 KB 2024-11-19T07:13:33,656 WARN [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,qqq,1732000407304.2df93dc30d9c55d765ada1f50e320789., aborting... org.apache.hadoop.hbase.regionserver.RegionServerStoppedException: Server db0afee3eab9,42435,1732000404708 stopping at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1466) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T07:13:33,656 ERROR [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2440): ***** ABORTING region server db0afee3eab9,42435,1732000404708: Failed to open region test,qqq,1732000407304.2df93dc30d9c55d765ada1f50e320789. and can not recover ***** org.apache.hadoop.hbase.regionserver.RegionServerStoppedException: Server db0afee3eab9,42435,1732000404708 stopping at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1466) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T07:13:33,656 ERROR [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-19T07:13:33,657 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-19T07:13:33,658 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-19T07:13:33,661 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-19T07:13:33,663 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-19T07:13:33,668 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/74657be6232ad22aaca548d82491de41/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-19T07:13:33,669 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/02309bab61f6207d73b9f19985e1fdb9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T07:13:33,669 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1732000407304.74657be6232ad22aaca548d82491de41. 2024-11-19T07:13:33,669 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 74657be6232ad22aaca548d82491de41: Waiting for close lock at 1732000413653Running coprocessor pre-close hooks at 1732000413653Disabling compacts and flushes for region at 1732000413653Disabling writes for close at 1732000413654 (+1 ms)Writing region close event to WAL at 1732000413654Running coprocessor post-close hooks at 1732000413669 (+15 ms)Closed at 1732000413669 2024-11-19T07:13:33,669 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1732000407304.74657be6232ad22aaca548d82491de41. 2024-11-19T07:13:33,670 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bbb518f4693490310eca052bf28f87fc, disabling compactions & flushes 2024-11-19T07:13:33,670 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1732000407304.bbb518f4693490310eca052bf28f87fc. 2024-11-19T07:13:33,670 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732000407304.bbb518f4693490310eca052bf28f87fc. 2024-11-19T07:13:33,670 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732000407304.bbb518f4693490310eca052bf28f87fc. after waiting 0 ms 2024-11-19T07:13:33,670 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732000407304.bbb518f4693490310eca052bf28f87fc. 2024-11-19T07:13:33,672 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1732000407304.02309bab61f6207d73b9f19985e1fdb9. 2024-11-19T07:13:33,672 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 02309bab61f6207d73b9f19985e1fdb9: Waiting for close lock at 1732000413653Running coprocessor pre-close hooks at 1732000413653Disabling compacts and flushes for region at 1732000413653Disabling writes for close at 1732000413654 (+1 ms)Writing region close event to WAL at 1732000413654Running coprocessor post-close hooks at 1732000413671 (+17 ms)Closed at 1732000413671 2024-11-19T07:13:33,672 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1732000407304.02309bab61f6207d73b9f19985e1fdb9. 2024-11-19T07:13:33,672 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7d9d31e1a2abc752a2056ed0e3a3fcec, disabling compactions & flushes 2024-11-19T07:13:33,672 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1732000407304.7d9d31e1a2abc752a2056ed0e3a3fcec. 2024-11-19T07:13:33,672 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732000407304.7d9d31e1a2abc752a2056ed0e3a3fcec. 2024-11-19T07:13:33,672 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732000407304.7d9d31e1a2abc752a2056ed0e3a3fcec. after waiting 0 ms 2024-11-19T07:13:33,672 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732000407304.7d9d31e1a2abc752a2056ed0e3a3fcec. 2024-11-19T07:13:33,672 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/.tmp/info/d8af03f406394058ab050f80d73793d6 is 138, key is test,ccc,1732000407304.7d9d31e1a2abc752a2056ed0e3a3fcec./info:regioninfo/1732000412231/Put/seqid=0 2024-11-19T07:13:33,676 INFO [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 506324664 }, "NonHeapMemoryUsage": { "committed": 163250176, "init": 7667712, "max": -1, "used": 160685536 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "db0afee3eab9", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 2, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 39169, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 116, "ProcessCallTime_min": 0, "ProcessCallTime_max": 18, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 4, "ProcessCallTime_median": 9, "ProcessCallTime_75th_percentile": 13, "ProcessCallTime_90th_percentile": 16, "ProcessCallTime_95th_percentile": 17, "ProcessCallTime_98th_percentile": 17, "ProcessCallTime_99th_percentile": 17, "ProcessCallTime_99.9th_percentile": 17, "ProcessCallTime_TimeRangeCount_0-1": 116, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 116, "QueueCallTime_min": 0, "QueueCallTime_max": 14, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 3, "QueueCallTime_median": 7, "QueueCallTime_75th_percentile": 10, "QueueCallTime_90th_percentile": 12, "QueueCallTime_95th_percentile": 13, "QueueCallTime_98th_percentile": 13, "QueueCallTime_99th_percentile": 13, "QueueCallTime_99.9th_percentile": 13, "QueueCallTime_TimeRangeCount_0-1": 116, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 116, "TotalCallTime_min": 0, "TotalCallTime_max": 21, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 5, "TotalCallTime_median": 10, "TotalCallTime_75th_percentile": 15, "TotalCallTime_90th_percentile": 18, "TotalCallTime_95th_percentile": 19, "TotalCallTime_98th_percentile": 20, "TotalCallTime_99th_percentile": 20, "TotalCallTime_99.9th_percentile": 20, "TotalCallTime_TimeRangeCount_0-1": 116, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 116, "ResponseSize_min": 0, "ResponseSize_max": 22359, "ResponseSize_mean": 589, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16769, "ResponseSize_90th_percentile": 20123, "ResponseSize_95th_percentile": 21241, "ResponseSize_98th_percentile": 21911, "ResponseSize_99th_percentile": 22135, "ResponseSize_99.9th_percentile": 22336, "ResponseSize_SizeRangeCount_0-10": 116, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 116, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 305, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 116, "sentBytes": 54892 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "db0afee3eab9", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:53655", "tag.serverName": "db0afee3eab9,37853,1732000404756", "tag.clusterId": "1a1c4a15-1b2b-4e90-9845-b6cc07547cfd", "tag.Context": "regionserver", "tag.Hostname": "db0afee3eab9", "regionCount": 9, "storeCount": 12, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 30111, "memStoreHeapSize": 53936, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732000404756, "averageRegionSize": 3345, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3332, "localBytesRead": 3332, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 18.8, "writeRequestRatePerSecond": 17.2, "ByteBuffAllocatorHeapAllocationBytes": 7456, "ByteBuffAllocatorPoolAllocationBytes": 465920, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 79, "totalRowActionRequestCount": 180, "readRequestCount": 94, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 86, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 16, "rpcMultiRequestCount": 1, "rpcMutateRequestCount": 60, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 16, "ScanTime_min": 0, "ScanTime_max": 10, "ScanTime_mean": 1, "ScanTime_25th_percentile": 2, "ScanTime_median": 5, "ScanTime_75th_percentile": 7, "ScanTime_90th_percentile": 9, "ScanTime_95th_percentile": 9, "ScanTime_98th_percentile": 9, "ScanTime_99th_percentile": 9, "ScanTime_99.9th_percentile": 9, "ScanTime_TimeRangeCount_0-1": 16, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 60, "Put_min": 1, "Put_max": 7, "Put_mean": 2, "Put_25th_percentile": 2, "Put_median": 4, "Put_75th_percentile": 5, "Put_90th_percentile": 6, "Put_95th_percentile": 6, "Put_98th_percentile": 6, "Put_99th_percentile": 6, "Put_99.9th_percentile": 6, "Put_TimeRangeCount_0-1": 60, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 1, "PutBatch_min": 10, "PutBatch_max": 10, "PutBatch_mean": 10, "PutBatch_25th_percentile": 10, "PutBatch_median": 10, "PutBatch_75th_percentile": 10, "PutBatch_90th_percentile": 10, "PutBatch_95th_percentile": 10, "PutBatch_98th_percentile": 10, "PutBatch_99th_percentile": 10, "PutBatch_99.9th_percentile": 10, "PutBatch_TimeRangeCount_0-1": 1, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 1, "Get_max": 2, "Get_mean": 1, "Get_25th_percentile": 1, "Get_median": 1, "Get_75th_percentile": 1, "Get_90th_percentile": 1, "Get_95th_percentile": 1, "Get_98th_percentile": 1, "Get_99th_percentile": 1, "Get_99.9th_percentile": 1, "Get_TimeRangeCount_0-1": 2, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 16, "ScanSize_min": 0, "ScanSize_max": 22232, "ScanSize_mean": 4186, "ScanSize_25th_percentile": 5558, "ScanSize_median": 11116, "ScanSize_75th_percentile": 16674, "ScanSize_90th_percentile": 20008, "ScanSize_95th_percentile": 21120, "ScanSize_98th_percentile": 21787, "ScanSize_99th_percentile": 22009, "ScanSize_99.9th_percentile": 22209, "ScanSize_SizeRangeCount_0-10": 16, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-19T07:13:33,680 WARN [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=db0afee3eab9:43779 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.handleException(AssignRegionHandler.java:173) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:106) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=db0afee3eab9:43779 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more 2024-11-19T07:13:33,682 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/bbb518f4693490310eca052bf28f87fc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T07:13:33,682 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/7d9d31e1a2abc752a2056ed0e3a3fcec/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T07:13:33,683 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1732000407304.bbb518f4693490310eca052bf28f87fc. 2024-11-19T07:13:33,683 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bbb518f4693490310eca052bf28f87fc: Waiting for close lock at 1732000413670Running coprocessor pre-close hooks at 1732000413670Disabling compacts and flushes for region at 1732000413670Disabling writes for close at 1732000413670Writing region close event to WAL at 1732000413676 (+6 ms)Running coprocessor post-close hooks at 1732000413683 (+7 ms)Closed at 1732000413683 2024-11-19T07:13:33,683 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1732000407304.7d9d31e1a2abc752a2056ed0e3a3fcec. 2024-11-19T07:13:33,683 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1732000407304.bbb518f4693490310eca052bf28f87fc. 2024-11-19T07:13:33,683 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7d9d31e1a2abc752a2056ed0e3a3fcec: Waiting for close lock at 1732000413672Running coprocessor pre-close hooks at 1732000413672Disabling compacts and flushes for region at 1732000413672Disabling writes for close at 1732000413672Writing region close event to WAL at 1732000413676 (+4 ms)Running coprocessor post-close hooks at 1732000413683 (+7 ms)Closed at 1732000413683 2024-11-19T07:13:33,683 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e880c03f4c433baa6242dd65ef2d8486, disabling compactions & flushes 2024-11-19T07:13:33,683 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1732000407304.7d9d31e1a2abc752a2056ed0e3a3fcec. 2024-11-19T07:13:33,683 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1732000407304.e880c03f4c433baa6242dd65ef2d8486. 2024-11-19T07:13:33,683 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732000407304.e880c03f4c433baa6242dd65ef2d8486. 2024-11-19T07:13:33,683 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 66fe58970f9c4a1cba9c3e3da843ed3d, disabling compactions & flushes 2024-11-19T07:13:33,683 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732000407304.e880c03f4c433baa6242dd65ef2d8486. after waiting 0 ms 2024-11-19T07:13:33,683 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1732000407304.66fe58970f9c4a1cba9c3e3da843ed3d. 2024-11-19T07:13:33,684 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732000407304.e880c03f4c433baa6242dd65ef2d8486. 2024-11-19T07:13:33,684 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732000407304.66fe58970f9c4a1cba9c3e3da843ed3d. 2024-11-19T07:13:33,684 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732000407304.66fe58970f9c4a1cba9c3e3da843ed3d. after waiting 0 ms 2024-11-19T07:13:33,684 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732000407304.66fe58970f9c4a1cba9c3e3da843ed3d. 2024-11-19T07:13:33,693 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/e880c03f4c433baa6242dd65ef2d8486/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T07:13:33,693 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/66fe58970f9c4a1cba9c3e3da843ed3d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T07:13:33,694 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1732000407304.66fe58970f9c4a1cba9c3e3da843ed3d. 2024-11-19T07:13:33,694 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1732000407304.e880c03f4c433baa6242dd65ef2d8486. 2024-11-19T07:13:33,694 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 66fe58970f9c4a1cba9c3e3da843ed3d: Waiting for close lock at 1732000413683Running coprocessor pre-close hooks at 1732000413683Disabling compacts and flushes for region at 1732000413683Disabling writes for close at 1732000413684 (+1 ms)Writing region close event to WAL at 1732000413689 (+5 ms)Running coprocessor post-close hooks at 1732000413694 (+5 ms)Closed at 1732000413694 2024-11-19T07:13:33,694 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e880c03f4c433baa6242dd65ef2d8486: Waiting for close lock at 1732000413683Running coprocessor pre-close hooks at 1732000413683Disabling compacts and flushes for region at 1732000413683Disabling writes for close at 1732000413683Writing region close event to WAL at 1732000413688 (+5 ms)Running coprocessor post-close hooks at 1732000413694 (+6 ms)Closed at 1732000413694 2024-11-19T07:13:33,694 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1732000407304.66fe58970f9c4a1cba9c3e3da843ed3d. 2024-11-19T07:13:33,694 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1732000407304.e880c03f4c433baa6242dd65ef2d8486. 2024-11-19T07:13:33,694 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 611b07fbb4fb5e3ea7255d9fa0f9fe40, disabling compactions & flushes 2024-11-19T07:13:33,694 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1732000407304.611b07fbb4fb5e3ea7255d9fa0f9fe40. 2024-11-19T07:13:33,694 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dba97405d3b35647d42d266df5e81cec, disabling compactions & flushes 2024-11-19T07:13:33,694 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732000407304.611b07fbb4fb5e3ea7255d9fa0f9fe40. 2024-11-19T07:13:33,694 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1732000407304.dba97405d3b35647d42d266df5e81cec. 2024-11-19T07:13:33,694 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1732000407304.611b07fbb4fb5e3ea7255d9fa0f9fe40. after waiting 0 ms 2024-11-19T07:13:33,694 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732000407304.dba97405d3b35647d42d266df5e81cec. 2024-11-19T07:13:33,694 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1732000407304.611b07fbb4fb5e3ea7255d9fa0f9fe40. 2024-11-19T07:13:33,694 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732000407304.dba97405d3b35647d42d266df5e81cec. after waiting 0 ms 2024-11-19T07:13:33,694 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732000407304.dba97405d3b35647d42d266df5e81cec. 2024-11-19T07:13:33,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741869_1045 (size=8195) 2024-11-19T07:13:33,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741869_1045 (size=8195) 2024-11-19T07:13:33,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741869_1045 (size=8195) 2024-11-19T07:13:33,698 INFO [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.66 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/.tmp/info/d8af03f406394058ab050f80d73793d6 2024-11-19T07:13:33,698 INFO [regionserver/db0afee3eab9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T07:13:33,700 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/611b07fbb4fb5e3ea7255d9fa0f9fe40/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T07:13:33,701 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/dba97405d3b35647d42d266df5e81cec/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T07:13:33,701 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1732000407304.611b07fbb4fb5e3ea7255d9fa0f9fe40. 2024-11-19T07:13:33,701 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 611b07fbb4fb5e3ea7255d9fa0f9fe40: Waiting for close lock at 1732000413694Running coprocessor pre-close hooks at 1732000413694Disabling compacts and flushes for region at 1732000413694Disabling writes for close at 1732000413694Writing region close event to WAL at 1732000413695 (+1 ms)Running coprocessor post-close hooks at 1732000413701 (+6 ms)Closed at 1732000413701 2024-11-19T07:13:33,702 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1732000407304.611b07fbb4fb5e3ea7255d9fa0f9fe40. 2024-11-19T07:13:33,702 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing db4ba820395e8b4a2b38d57f8a222e97, disabling compactions & flushes 2024-11-19T07:13:33,702 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1732000407304.db4ba820395e8b4a2b38d57f8a222e97. 2024-11-19T07:13:33,702 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732000407304.db4ba820395e8b4a2b38d57f8a222e97. 2024-11-19T07:13:33,702 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732000407304.db4ba820395e8b4a2b38d57f8a222e97. after waiting 0 ms 2024-11-19T07:13:33,702 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732000407304.db4ba820395e8b4a2b38d57f8a222e97. 2024-11-19T07:13:33,703 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1732000407304.dba97405d3b35647d42d266df5e81cec. 2024-11-19T07:13:33,703 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dba97405d3b35647d42d266df5e81cec: Waiting for close lock at 1732000413694Running coprocessor pre-close hooks at 1732000413694Disabling compacts and flushes for region at 1732000413694Disabling writes for close at 1732000413694Writing region close event to WAL at 1732000413696 (+2 ms)Running coprocessor post-close hooks at 1732000413702 (+6 ms)Closed at 1732000413703 (+1 ms) 2024-11-19T07:13:33,703 INFO [regionserver/db0afee3eab9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T07:13:33,703 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1732000407304.dba97405d3b35647d42d266df5e81cec. 2024-11-19T07:13:33,703 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9357ae6c6e8337f25dacbf0a10701ef6, disabling compactions & flushes 2024-11-19T07:13:33,703 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1732000407304.9357ae6c6e8337f25dacbf0a10701ef6. 2024-11-19T07:13:33,703 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732000407304.9357ae6c6e8337f25dacbf0a10701ef6. 2024-11-19T07:13:33,703 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732000407304.9357ae6c6e8337f25dacbf0a10701ef6. after waiting 0 ms 2024-11-19T07:13:33,703 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732000407304.9357ae6c6e8337f25dacbf0a10701ef6. 2024-11-19T07:13:33,708 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/.tmp/info/d8af03f406394058ab050f80d73793d6 as hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/info/d8af03f406394058ab050f80d73793d6 2024-11-19T07:13:33,709 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/db4ba820395e8b4a2b38d57f8a222e97/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T07:13:33,709 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/9357ae6c6e8337f25dacbf0a10701ef6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T07:13:33,710 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1732000407304.db4ba820395e8b4a2b38d57f8a222e97. 2024-11-19T07:13:33,710 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for db4ba820395e8b4a2b38d57f8a222e97: Waiting for close lock at 1732000413702Running coprocessor pre-close hooks at 1732000413702Disabling compacts and flushes for region at 1732000413702Disabling writes for close at 1732000413702Writing region close event to WAL at 1732000413703 (+1 ms)Running coprocessor post-close hooks at 1732000413710 (+7 ms)Closed at 1732000413710 2024-11-19T07:13:33,710 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1732000407304.db4ba820395e8b4a2b38d57f8a222e97. 2024-11-19T07:13:33,710 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d703151d82f3a429f348967988283542, disabling compactions & flushes 2024-11-19T07:13:33,710 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1732000407304.9357ae6c6e8337f25dacbf0a10701ef6. 2024-11-19T07:13:33,710 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1732000407304.d703151d82f3a429f348967988283542. 2024-11-19T07:13:33,710 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9357ae6c6e8337f25dacbf0a10701ef6: Waiting for close lock at 1732000413703Running coprocessor pre-close hooks at 1732000413703Disabling compacts and flushes for region at 1732000413703Disabling writes for close at 1732000413703Writing region close event to WAL at 1732000413704 (+1 ms)Running coprocessor post-close hooks at 1732000413710 (+6 ms)Closed at 1732000413710 2024-11-19T07:13:33,711 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732000407304.d703151d82f3a429f348967988283542. 2024-11-19T07:13:33,711 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1732000407304.9357ae6c6e8337f25dacbf0a10701ef6. 2024-11-19T07:13:33,711 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732000407304.d703151d82f3a429f348967988283542. after waiting 0 ms 2024-11-19T07:13:33,711 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732000407304.d703151d82f3a429f348967988283542. 2024-11-19T07:13:33,711 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 12dcc969f79695b5f0d224b605268d2d, disabling compactions & flushes 2024-11-19T07:13:33,711 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1732000407304.12dcc969f79695b5f0d224b605268d2d. 2024-11-19T07:13:33,711 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732000407304.12dcc969f79695b5f0d224b605268d2d. 2024-11-19T07:13:33,711 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732000407304.12dcc969f79695b5f0d224b605268d2d. after waiting 0 ms 2024-11-19T07:13:33,711 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732000407304.12dcc969f79695b5f0d224b605268d2d. 2024-11-19T07:13:33,715 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/d703151d82f3a429f348967988283542/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T07:13:33,716 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1732000407304.d703151d82f3a429f348967988283542. 2024-11-19T07:13:33,716 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d703151d82f3a429f348967988283542: Waiting for close lock at 1732000413710Running coprocessor pre-close hooks at 1732000413710Disabling compacts and flushes for region at 1732000413710Disabling writes for close at 1732000413711 (+1 ms)Writing region close event to WAL at 1732000413711Running coprocessor post-close hooks at 1732000413716 (+5 ms)Closed at 1732000413716 2024-11-19T07:13:33,716 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/12dcc969f79695b5f0d224b605268d2d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T07:13:33,716 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1732000407304.d703151d82f3a429f348967988283542. 2024-11-19T07:13:33,716 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 32d987cca115e6f476cdb5686be2b726, disabling compactions & flushes 2024-11-19T07:13:33,716 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1732000407304.32d987cca115e6f476cdb5686be2b726. 2024-11-19T07:13:33,717 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732000407304.32d987cca115e6f476cdb5686be2b726. 2024-11-19T07:13:33,717 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732000407304.32d987cca115e6f476cdb5686be2b726. after waiting 0 ms 2024-11-19T07:13:33,717 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732000407304.32d987cca115e6f476cdb5686be2b726. 2024-11-19T07:13:33,717 INFO [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/info/d8af03f406394058ab050f80d73793d6, entries=24, sequenceid=75, filesize=8.0 K 2024-11-19T07:13:33,718 INFO [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 63ms, sequenceid=75, compaction requested=false 2024-11-19T07:13:33,718 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1732000407304.12dcc969f79695b5f0d224b605268d2d. 2024-11-19T07:13:33,718 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 12dcc969f79695b5f0d224b605268d2d: Waiting for close lock at 1732000413711Running coprocessor pre-close hooks at 1732000413711Disabling compacts and flushes for region at 1732000413711Disabling writes for close at 1732000413711Writing region close event to WAL at 1732000413712 (+1 ms)Running coprocessor post-close hooks at 1732000413718 (+6 ms)Closed at 1732000413718 2024-11-19T07:13:33,719 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1732000407304.12dcc969f79695b5f0d224b605268d2d. 2024-11-19T07:13:33,719 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fdf73eccbee4f01f5a86b87f17dcdb8b, disabling compactions & flushes 2024-11-19T07:13:33,719 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1732000407304.fdf73eccbee4f01f5a86b87f17dcdb8b. 2024-11-19T07:13:33,719 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732000407304.fdf73eccbee4f01f5a86b87f17dcdb8b. 2024-11-19T07:13:33,719 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732000407304.fdf73eccbee4f01f5a86b87f17dcdb8b. after waiting 0 ms 2024-11-19T07:13:33,719 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732000407304.fdf73eccbee4f01f5a86b87f17dcdb8b. 2024-11-19T07:13:33,723 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/32d987cca115e6f476cdb5686be2b726/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T07:13:33,723 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1732000407304.32d987cca115e6f476cdb5686be2b726. 2024-11-19T07:13:33,723 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 32d987cca115e6f476cdb5686be2b726: Waiting for close lock at 1732000413716Running coprocessor pre-close hooks at 1732000413716Disabling compacts and flushes for region at 1732000413716Disabling writes for close at 1732000413717 (+1 ms)Writing region close event to WAL at 1732000413717Running coprocessor post-close hooks at 1732000413723 (+6 ms)Closed at 1732000413723 2024-11-19T07:13:33,724 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1732000407304.32d987cca115e6f476cdb5686be2b726. 2024-11-19T07:13:33,724 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 08f6487d04261d6f0713b4b5adeacf2e, disabling compactions & flushes 2024-11-19T07:13:33,724 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1732000407304.08f6487d04261d6f0713b4b5adeacf2e. 2024-11-19T07:13:33,724 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732000407304.08f6487d04261d6f0713b4b5adeacf2e. 2024-11-19T07:13:33,724 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732000407304.08f6487d04261d6f0713b4b5adeacf2e. after waiting 0 ms 2024-11-19T07:13:33,724 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732000407304.08f6487d04261d6f0713b4b5adeacf2e. 2024-11-19T07:13:33,727 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/fdf73eccbee4f01f5a86b87f17dcdb8b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T07:13:33,727 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/hbase/meta/1588230740/recovered.edits/78.seqid, newMaxSeqId=78, maxSeqId=64 2024-11-19T07:13:33,727 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1732000407304.fdf73eccbee4f01f5a86b87f17dcdb8b. 2024-11-19T07:13:33,727 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fdf73eccbee4f01f5a86b87f17dcdb8b: Waiting for close lock at 1732000413719Running coprocessor pre-close hooks at 1732000413719Disabling compacts and flushes for region at 1732000413719Disabling writes for close at 1732000413719Writing region close event to WAL at 1732000413719Running coprocessor post-close hooks at 1732000413727 (+8 ms)Closed at 1732000413727 2024-11-19T07:13:33,728 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1732000407304.fdf73eccbee4f01f5a86b87f17dcdb8b. 2024-11-19T07:13:33,728 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T07:13:33,728 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2f38687f659a16d9e2a781f2a75d514a, disabling compactions & flushes 2024-11-19T07:13:33,728 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1732000407304.2f38687f659a16d9e2a781f2a75d514a. 2024-11-19T07:13:33,728 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732000407304.2f38687f659a16d9e2a781f2a75d514a. 2024-11-19T07:13:33,728 INFO [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T07:13:33,728 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732000407304.2f38687f659a16d9e2a781f2a75d514a. after waiting 0 ms 2024-11-19T07:13:33,728 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732000407304.2f38687f659a16d9e2a781f2a75d514a. 2024-11-19T07:13:33,728 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732000413655Running coprocessor pre-close hooks at 1732000413655Disabling compacts and flushes for region at 1732000413655Disabling writes for close at 1732000413655Obtaining lock to block concurrent updates at 1732000413655Preparing flush snapshotting stores in 1588230740 at 1732000413655Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2720, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1732000413662 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732000413663 (+1 ms)Flushing 1588230740/info: creating writer at 1732000413663Flushing 1588230740/info: appending metadata at 1732000413672 (+9 ms)Flushing 1588230740/info: closing flushed file at 1732000413672Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@398ffaf4: reopening flushed file at 1732000413707 (+35 ms)Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 63ms, sequenceid=75, compaction requested=false at 1732000413718 (+11 ms)Writing region close event to WAL at 1732000413721 (+3 ms)Running coprocessor post-close hooks at 1732000413728 (+7 ms)Closed at 1732000413728 2024-11-19T07:13:33,728 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-19T07:13:33,730 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/08f6487d04261d6f0713b4b5adeacf2e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T07:13:33,731 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1732000407304.08f6487d04261d6f0713b4b5adeacf2e. 2024-11-19T07:13:33,731 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 08f6487d04261d6f0713b4b5adeacf2e: Waiting for close lock at 1732000413724Running coprocessor pre-close hooks at 1732000413724Disabling compacts and flushes for region at 1732000413724Disabling writes for close at 1732000413724Writing region close event to WAL at 1732000413726 (+2 ms)Running coprocessor post-close hooks at 1732000413730 (+4 ms)Closed at 1732000413730 2024-11-19T07:13:33,731 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1732000407304.08f6487d04261d6f0713b4b5adeacf2e. 2024-11-19T07:13:33,731 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d2ef4078aa4f5f4b6f4dc7188dd2f1cb, disabling compactions & flushes 2024-11-19T07:13:33,731 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1732000407304.d2ef4078aa4f5f4b6f4dc7188dd2f1cb. 2024-11-19T07:13:33,731 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732000407304.d2ef4078aa4f5f4b6f4dc7188dd2f1cb. 2024-11-19T07:13:33,731 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732000407304.d2ef4078aa4f5f4b6f4dc7188dd2f1cb. after waiting 0 ms 2024-11-19T07:13:33,731 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732000407304.d2ef4078aa4f5f4b6f4dc7188dd2f1cb. 2024-11-19T07:13:33,732 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/2f38687f659a16d9e2a781f2a75d514a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T07:13:33,733 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1732000407304.2f38687f659a16d9e2a781f2a75d514a. 2024-11-19T07:13:33,733 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2f38687f659a16d9e2a781f2a75d514a: Waiting for close lock at 1732000413728Running coprocessor pre-close hooks at 1732000413728Disabling compacts and flushes for region at 1732000413728Disabling writes for close at 1732000413728Writing region close event to WAL at 1732000413729 (+1 ms)Running coprocessor post-close hooks at 1732000413733 (+4 ms)Closed at 1732000413733 2024-11-19T07:13:33,733 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1732000407304.2f38687f659a16d9e2a781f2a75d514a. 2024-11-19T07:13:33,733 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 126b77a274d2ba63b12dd23c9e7ababb, disabling compactions & flushes 2024-11-19T07:13:33,733 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1732000407304.126b77a274d2ba63b12dd23c9e7ababb. 2024-11-19T07:13:33,733 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732000407304.126b77a274d2ba63b12dd23c9e7ababb. 2024-11-19T07:13:33,734 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732000407304.126b77a274d2ba63b12dd23c9e7ababb. after waiting 0 ms 2024-11-19T07:13:33,734 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732000407304.126b77a274d2ba63b12dd23c9e7ababb. 2024-11-19T07:13:33,735 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/d2ef4078aa4f5f4b6f4dc7188dd2f1cb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T07:13:33,736 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1732000407304.d2ef4078aa4f5f4b6f4dc7188dd2f1cb. 2024-11-19T07:13:33,736 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d2ef4078aa4f5f4b6f4dc7188dd2f1cb: Waiting for close lock at 1732000413731Running coprocessor pre-close hooks at 1732000413731Disabling compacts and flushes for region at 1732000413731Disabling writes for close at 1732000413731Writing region close event to WAL at 1732000413731Running coprocessor post-close hooks at 1732000413736 (+5 ms)Closed at 1732000413736 2024-11-19T07:13:33,736 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1732000407304.d2ef4078aa4f5f4b6f4dc7188dd2f1cb. 2024-11-19T07:13:33,736 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3a637105eb1d8b75696b3cfabe5d8173, disabling compactions & flushes 2024-11-19T07:13:33,736 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1732000407304.3a637105eb1d8b75696b3cfabe5d8173. 2024-11-19T07:13:33,737 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732000407304.3a637105eb1d8b75696b3cfabe5d8173. 2024-11-19T07:13:33,737 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732000407304.3a637105eb1d8b75696b3cfabe5d8173. after waiting 0 ms 2024-11-19T07:13:33,737 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732000407304.3a637105eb1d8b75696b3cfabe5d8173. 2024-11-19T07:13:33,738 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/126b77a274d2ba63b12dd23c9e7ababb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T07:13:33,739 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1732000407304.126b77a274d2ba63b12dd23c9e7ababb. 2024-11-19T07:13:33,739 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 126b77a274d2ba63b12dd23c9e7ababb: Waiting for close lock at 1732000413733Running coprocessor pre-close hooks at 1732000413733Disabling compacts and flushes for region at 1732000413733Disabling writes for close at 1732000413734 (+1 ms)Writing region close event to WAL at 1732000413734Running coprocessor post-close hooks at 1732000413739 (+5 ms)Closed at 1732000413739 2024-11-19T07:13:33,739 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1732000407304.126b77a274d2ba63b12dd23c9e7ababb. 2024-11-19T07:13:33,740 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 49b850b07eedb76caa94e4c93ac5c832, disabling compactions & flushes 2024-11-19T07:13:33,740 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1732000407304.49b850b07eedb76caa94e4c93ac5c832. 2024-11-19T07:13:33,740 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732000407304.49b850b07eedb76caa94e4c93ac5c832. 2024-11-19T07:13:33,740 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732000407304.49b850b07eedb76caa94e4c93ac5c832. after waiting 0 ms 2024-11-19T07:13:33,740 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732000407304.49b850b07eedb76caa94e4c93ac5c832. 2024-11-19T07:13:33,741 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/3a637105eb1d8b75696b3cfabe5d8173/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T07:13:33,742 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1732000407304.3a637105eb1d8b75696b3cfabe5d8173. 2024-11-19T07:13:33,742 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3a637105eb1d8b75696b3cfabe5d8173: Waiting for close lock at 1732000413736Running coprocessor pre-close hooks at 1732000413736Disabling compacts and flushes for region at 1732000413736Disabling writes for close at 1732000413737 (+1 ms)Writing region close event to WAL at 1732000413737Running coprocessor post-close hooks at 1732000413742 (+5 ms)Closed at 1732000413742 2024-11-19T07:13:33,742 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1732000407304.3a637105eb1d8b75696b3cfabe5d8173. 2024-11-19T07:13:33,743 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4421c1189c5fdff3a40dcc58728c41ab, disabling compactions & flushes 2024-11-19T07:13:33,743 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1732000407304.4421c1189c5fdff3a40dcc58728c41ab. 2024-11-19T07:13:33,743 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732000407304.4421c1189c5fdff3a40dcc58728c41ab. 2024-11-19T07:13:33,743 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732000407304.4421c1189c5fdff3a40dcc58728c41ab. after waiting 0 ms 2024-11-19T07:13:33,743 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732000407304.4421c1189c5fdff3a40dcc58728c41ab. 2024-11-19T07:13:33,745 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/49b850b07eedb76caa94e4c93ac5c832/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T07:13:33,745 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1732000407304.49b850b07eedb76caa94e4c93ac5c832. 2024-11-19T07:13:33,746 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 49b850b07eedb76caa94e4c93ac5c832: Waiting for close lock at 1732000413740Running coprocessor pre-close hooks at 1732000413740Disabling compacts and flushes for region at 1732000413740Disabling writes for close at 1732000413740Writing region close event to WAL at 1732000413740Running coprocessor post-close hooks at 1732000413745 (+5 ms)Closed at 1732000413745 2024-11-19T07:13:33,746 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1732000407304.49b850b07eedb76caa94e4c93ac5c832. 2024-11-19T07:13:33,746 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 82941673e9ad03e5a037091750c3f315, disabling compactions & flushes 2024-11-19T07:13:33,746 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1732000407304.82941673e9ad03e5a037091750c3f315. 2024-11-19T07:13:33,746 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732000407304.82941673e9ad03e5a037091750c3f315. 2024-11-19T07:13:33,746 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732000407304.82941673e9ad03e5a037091750c3f315. after waiting 0 ms 2024-11-19T07:13:33,746 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732000407304.82941673e9ad03e5a037091750c3f315. 2024-11-19T07:13:33,747 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/4421c1189c5fdff3a40dcc58728c41ab/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T07:13:33,748 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1732000407304.4421c1189c5fdff3a40dcc58728c41ab. 2024-11-19T07:13:33,748 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4421c1189c5fdff3a40dcc58728c41ab: Waiting for close lock at 1732000413743Running coprocessor pre-close hooks at 1732000413743Disabling compacts and flushes for region at 1732000413743Disabling writes for close at 1732000413743Writing region close event to WAL at 1732000413743Running coprocessor post-close hooks at 1732000413748 (+5 ms)Closed at 1732000413748 2024-11-19T07:13:33,749 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1732000407304.4421c1189c5fdff3a40dcc58728c41ab. 2024-11-19T07:13:33,749 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1ece759217c055d9a295034ce69dd86d, disabling compactions & flushes 2024-11-19T07:13:33,749 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1732000407304.1ece759217c055d9a295034ce69dd86d. 2024-11-19T07:13:33,749 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732000407304.1ece759217c055d9a295034ce69dd86d. 2024-11-19T07:13:33,749 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732000407304.1ece759217c055d9a295034ce69dd86d. after waiting 0 ms 2024-11-19T07:13:33,749 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732000407304.1ece759217c055d9a295034ce69dd86d. 2024-11-19T07:13:33,751 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/82941673e9ad03e5a037091750c3f315/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T07:13:33,751 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1732000407304.82941673e9ad03e5a037091750c3f315. 2024-11-19T07:13:33,751 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 82941673e9ad03e5a037091750c3f315: Waiting for close lock at 1732000413746Running coprocessor pre-close hooks at 1732000413746Disabling compacts and flushes for region at 1732000413746Disabling writes for close at 1732000413746Writing region close event to WAL at 1732000413746Running coprocessor post-close hooks at 1732000413751 (+5 ms)Closed at 1732000413751 2024-11-19T07:13:33,752 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1732000407304.82941673e9ad03e5a037091750c3f315. 2024-11-19T07:13:33,752 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 43676ec232db06c9bf4af64db1ede4f5, disabling compactions & flushes 2024-11-19T07:13:33,752 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1732000407304.43676ec232db06c9bf4af64db1ede4f5. 2024-11-19T07:13:33,752 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732000407304.43676ec232db06c9bf4af64db1ede4f5. 2024-11-19T07:13:33,752 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732000407304.43676ec232db06c9bf4af64db1ede4f5. after waiting 0 ms 2024-11-19T07:13:33,752 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732000407304.43676ec232db06c9bf4af64db1ede4f5. 2024-11-19T07:13:33,754 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/1ece759217c055d9a295034ce69dd86d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T07:13:33,755 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1732000407304.1ece759217c055d9a295034ce69dd86d. 2024-11-19T07:13:33,755 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1ece759217c055d9a295034ce69dd86d: Waiting for close lock at 1732000413749Running coprocessor pre-close hooks at 1732000413749Disabling compacts and flushes for region at 1732000413749Disabling writes for close at 1732000413749Writing region close event to WAL at 1732000413749Running coprocessor post-close hooks at 1732000413755 (+6 ms)Closed at 1732000413755 2024-11-19T07:13:33,755 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1732000407304.1ece759217c055d9a295034ce69dd86d. 2024-11-19T07:13:33,757 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/43676ec232db06c9bf4af64db1ede4f5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T07:13:33,757 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1732000407304.43676ec232db06c9bf4af64db1ede4f5. 2024-11-19T07:13:33,758 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 43676ec232db06c9bf4af64db1ede4f5: Waiting for close lock at 1732000413752Running coprocessor pre-close hooks at 1732000413752Disabling compacts and flushes for region at 1732000413752Disabling writes for close at 1732000413752Writing region close event to WAL at 1732000413752Running coprocessor post-close hooks at 1732000413757 (+5 ms)Closed at 1732000413757 2024-11-19T07:13:33,758 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1732000407304.43676ec232db06c9bf4af64db1ede4f5. 2024-11-19T07:13:33,758 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3aa6780c71c61c778946d2086357143d, disabling compactions & flushes 2024-11-19T07:13:33,758 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1732000407304.3aa6780c71c61c778946d2086357143d. 2024-11-19T07:13:33,758 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732000407304.3aa6780c71c61c778946d2086357143d. 2024-11-19T07:13:33,758 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732000407304.3aa6780c71c61c778946d2086357143d. after waiting 0 ms 2024-11-19T07:13:33,758 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732000407304.3aa6780c71c61c778946d2086357143d. 2024-11-19T07:13:33,762 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/data/default/test/3aa6780c71c61c778946d2086357143d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T07:13:33,763 INFO [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1732000407304.3aa6780c71c61c778946d2086357143d. 2024-11-19T07:13:33,763 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3aa6780c71c61c778946d2086357143d: Waiting for close lock at 1732000413758Running coprocessor pre-close hooks at 1732000413758Disabling compacts and flushes for region at 1732000413758Disabling writes for close at 1732000413758Writing region close event to WAL at 1732000413758Running coprocessor post-close hooks at 1732000413763 (+5 ms)Closed at 1732000413763 2024-11-19T07:13:33,763 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1732000407304.3aa6780c71c61c778946d2086357143d. 2024-11-19T07:13:33,853 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.HRegionServer(976): stopping server db0afee3eab9,34425,1732000404604; all regions closed. 2024-11-19T07:13:33,855 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.HRegionServer(976): stopping server db0afee3eab9,42435,1732000404708; all regions closed. 2024-11-19T07:13:33,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741833_1009 (size=7074) 2024-11-19T07:13:33,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741833_1009 (size=7074) 2024-11-19T07:13:33,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741833_1009 (size=7074) 2024-11-19T07:13:33,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741865_1041 (size=4204) 2024-11-19T07:13:33,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741865_1041 (size=4204) 2024-11-19T07:13:33,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741865_1041 (size=4204) 2024-11-19T07:13:33,862 DEBUG [RS:0;db0afee3eab9:34425 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/oldWALs 2024-11-19T07:13:33,862 INFO [RS:0;db0afee3eab9:34425 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL db0afee3eab9%2C34425%2C1732000404604:(num 1732000406456) 2024-11-19T07:13:33,862 DEBUG [RS:0;db0afee3eab9:34425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T07:13:33,862 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T07:13:33,863 INFO [RS:0;db0afee3eab9:34425 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T07:13:33,863 INFO [RS:0;db0afee3eab9:34425 {}] hbase.ChoreService(370): Chore service for: regionserver/db0afee3eab9:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-19T07:13:33,863 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T07:13:33,863 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T07:13:33,863 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T07:13:33,863 INFO [RS:0;db0afee3eab9:34425 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T07:13:33,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741834_1010 (size=6795) 2024-11-19T07:13:33,864 INFO [RS:0;db0afee3eab9:34425 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34425 2024-11-19T07:13:33,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741834_1010 (size=6795) 2024-11-19T07:13:33,864 INFO [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T07:13:33,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741834_1010 (size=6795) 2024-11-19T07:13:33,865 DEBUG [RS:1;db0afee3eab9:42435 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T07:13:33,865 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T07:13:33,866 INFO [RS:1;db0afee3eab9:42435 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T07:13:33,866 INFO [RS:1;db0afee3eab9:42435 {}] hbase.ChoreService(370): Chore service for: regionserver/db0afee3eab9:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-19T07:13:33,866 INFO [RS:1;db0afee3eab9:42435 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T07:13:33,866 INFO [RS:1;db0afee3eab9:42435 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42435 2024-11-19T07:13:33,866 INFO [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T07:13:33,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T07:13:33,869 INFO [RS:0;db0afee3eab9:34425 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T07:13:33,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/db0afee3eab9,34425,1732000404604 2024-11-19T07:13:33,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/db0afee3eab9,42435,1732000404708 2024-11-19T07:13:33,880 INFO [RS:1;db0afee3eab9:42435 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T07:13:33,880 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$370/0x00007fb81c901ad8@3c07b25d rejected from java.util.concurrent.ThreadPoolExecutor@79053bb9[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 39] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-11-19T07:13:33,890 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [db0afee3eab9,34425,1732000404604] 2024-11-19T07:13:33,911 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/db0afee3eab9,34425,1732000404604 already deleted, retry=false 2024-11-19T07:13:33,912 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; db0afee3eab9,34425,1732000404604 expired; onlineServers=1 2024-11-19T07:13:33,912 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [db0afee3eab9,42435,1732000404708] 2024-11-19T07:13:33,922 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/db0afee3eab9,42435,1732000404708 already deleted, retry=false 2024-11-19T07:13:33,922 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; db0afee3eab9,42435,1732000404708 expired; onlineServers=0 2024-11-19T07:13:33,922 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'db0afee3eab9,43779,1732000403834' ***** 2024-11-19T07:13:33,922 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-19T07:13:33,922 INFO [M:0;db0afee3eab9:43779 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T07:13:33,922 INFO [M:0;db0afee3eab9:43779 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T07:13:33,923 DEBUG [M:0;db0afee3eab9:43779 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-19T07:13:33,923 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-19T07:13:33,923 DEBUG [M:0;db0afee3eab9:43779 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-19T07:13:33,923 DEBUG [master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.large.0-1732000406034 {}] cleaner.HFileCleaner(306): Exit Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.large.0-1732000406034,5,FailOnTimeoutGroup] 2024-11-19T07:13:33,923 DEBUG [master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.small.0-1732000406035 {}] cleaner.HFileCleaner(306): Exit Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.small.0-1732000406035,5,FailOnTimeoutGroup] 2024-11-19T07:13:33,923 INFO [M:0;db0afee3eab9:43779 {}] hbase.ChoreService(370): Chore service for: master/db0afee3eab9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-19T07:13:33,923 INFO [M:0;db0afee3eab9:43779 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T07:13:33,924 DEBUG [M:0;db0afee3eab9:43779 {}] master.HMaster(1795): Stopping service threads 2024-11-19T07:13:33,924 INFO [M:0;db0afee3eab9:43779 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-19T07:13:33,924 INFO [M:0;db0afee3eab9:43779 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T07:13:33,925 INFO [M:0;db0afee3eab9:43779 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T07:13:33,925 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-19T07:13:33,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-19T07:13:33,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T07:13:33,933 DEBUG [M:0;db0afee3eab9:43779 {}] zookeeper.ZKUtil(347): master:43779-0x10151f0c9da0000, quorum=127.0.0.1:53655, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-19T07:13:33,933 WARN [M:0;db0afee3eab9:43779 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-19T07:13:33,934 INFO [M:0;db0afee3eab9:43779 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/.lastflushedseqids 2024-11-19T07:13:33,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741870_1046 (size=1696) 2024-11-19T07:13:33,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741870_1046 (size=1696) 2024-11-19T07:13:33,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741870_1046 (size=1696) 2024-11-19T07:13:33,953 INFO [M:0;db0afee3eab9:43779 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T07:13:33,953 INFO [M:0;db0afee3eab9:43779 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-19T07:13:33,953 DEBUG [M:0;db0afee3eab9:43779 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T07:13:33,953 INFO [M:0;db0afee3eab9:43779 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T07:13:33,953 DEBUG [M:0;db0afee3eab9:43779 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T07:13:33,953 DEBUG [M:0;db0afee3eab9:43779 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T07:13:33,953 DEBUG [M:0;db0afee3eab9:43779 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T07:13:33,954 INFO [M:0;db0afee3eab9:43779 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=259.52 KB heapSize=315.43 KB 2024-11-19T07:13:33,972 DEBUG [M:0;db0afee3eab9:43779 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8cb5a2ad55174698ba1a0c52e2f6b47c is 82, key is hbase:meta,,1/info:regioninfo/1732000411817/Put/seqid=0 2024-11-19T07:13:33,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44773 is added to blk_1073741871_1047 (size=6071) 2024-11-19T07:13:33,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41661 is added to blk_1073741871_1047 (size=6071) 2024-11-19T07:13:33,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39225 is added to blk_1073741871_1047 (size=6071) 2024-11-19T07:13:33,981 INFO [M:0;db0afee3eab9:43779 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8cb5a2ad55174698ba1a0c52e2f6b47c 2024-11-19T07:13:33,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T07:13:33,990 INFO [RS:0;db0afee3eab9:34425 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T07:13:33,991 INFO [RS:0;db0afee3eab9:34425 {}] regionserver.HRegionServer(1031): Exiting; stopping=db0afee3eab9,34425,1732000404604; zookeeper connection closed. 2024-11-19T07:13:33,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34425-0x10151f0c9da0001, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T07:13:33,991 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@197c614e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@197c614e 2024-11-19T07:13:34,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T07:13:34,001 INFO [RS:1;db0afee3eab9:42435 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T07:13:34,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42435-0x10151f0c9da0002, quorum=127.0.0.1:53655, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T07:13:34,001 INFO [RS:1;db0afee3eab9:42435 {}] regionserver.HRegionServer(1031): Exiting; stopping=db0afee3eab9,42435,1732000404708; zookeeper connection closed. 2024-11-19T07:13:34,001 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1ff67dde {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1ff67dde 2024-11-19T07:13:34,002 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-19T07:13:34,005 DEBUG [M:0;db0afee3eab9:43779 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38229/user/jenkins/test-data/b7e526d2-6c64-ba9d-5b35-7a9dc638e68d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8d666e329c2749b289d1b6293158727e is 2990, key is \x00\x00\x00\x00\x00\x00\x00