2024-12-03 06:21:51,641 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-12-03 06:21:51,657 main DEBUG Took 0.013563 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-03 06:21:51,658 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-03 06:21:51,658 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-03 06:21:51,660 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-03 06:21:51,661 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 06:21:51,668 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-03 06:21:51,685 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 06:21:51,687 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 06:21:51,688 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 06:21:51,688 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 06:21:51,689 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 06:21:51,689 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 06:21:51,691 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 06:21:51,691 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 06:21:51,692 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 06:21:51,692 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 06:21:51,693 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 06:21:51,694 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 06:21:51,694 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 06:21:51,695 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 06:21:51,695 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 06:21:51,695 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 06:21:51,696 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 06:21:51,696 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 06:21:51,696 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 06:21:51,697 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 06:21:51,697 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 06:21:51,697 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 06:21:51,698 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 06:21:51,698 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 06:21:51,699 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 06:21:51,699 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-03 06:21:51,701 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 06:21:51,703 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-03 06:21:51,705 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-03 06:21:51,706 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-03 06:21:51,708 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-03 06:21:51,709 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-03 06:21:51,721 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-03 06:21:51,724 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-03 06:21:51,727 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-03 06:21:51,727 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-03 06:21:51,728 main DEBUG createAppenders(={Console}) 2024-12-03 06:21:51,729 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-12-03 06:21:51,730 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-12-03 06:21:51,730 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-12-03 06:21:51,731 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-03 06:21:51,731 main DEBUG OutputStream closed 2024-12-03 06:21:51,731 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-03 06:21:51,732 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-03 06:21:51,732 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-12-03 06:21:51,828 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-03 06:21:51,832 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-03 06:21:51,833 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-03 06:21:51,835 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-03 06:21:51,836 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-03 06:21:51,836 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-03 06:21:51,837 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-03 06:21:51,837 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-03 06:21:51,838 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-03 06:21:51,838 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-03 06:21:51,838 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-03 06:21:51,839 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-03 06:21:51,839 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-03 06:21:51,840 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-03 06:21:51,840 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-03 06:21:51,840 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-03 06:21:51,841 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-03 06:21:51,842 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-03 06:21:51,845 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03 06:21:51,846 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-12-03 06:21:51,846 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-03 06:21:51,847 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-12-03T06:21:51,860 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-12-03 06:21:51,863 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-03 06:21:51,864 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03T06:21:52,066 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03 2024-12-03T06:21:52,084 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-03T06:21:52,104 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/cluster_9b3a36d6-da9f-80f6-d98c-a615b634f695, deleteOnExit=true 2024-12-03T06:21:52,105 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-03T06:21:52,106 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/test.cache.data in system properties and HBase conf 2024-12-03T06:21:52,106 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/hadoop.tmp.dir in system properties and HBase conf 2024-12-03T06:21:52,107 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/hadoop.log.dir in system properties and HBase conf 2024-12-03T06:21:52,107 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-03T06:21:52,108 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-03T06:21:52,108 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-03T06:21:52,201 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-03T06:21:52,306 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-03T06:21:52,310 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-03T06:21:52,310 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-03T06:21:52,311 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-03T06:21:52,312 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T06:21:52,312 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-03T06:21:52,313 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-03T06:21:52,313 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T06:21:52,313 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T06:21:52,314 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-03T06:21:52,314 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/nfs.dump.dir in system properties and HBase conf 2024-12-03T06:21:52,315 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/java.io.tmpdir in system properties and HBase conf 2024-12-03T06:21:52,315 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T06:21:52,315 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-03T06:21:52,316 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-03T06:21:53,338 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-03T06:21:53,439 INFO [Time-limited test {}] log.Log(170): Logging initialized @2516ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-03T06:21:53,523 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T06:21:53,576 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T06:21:53,595 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T06:21:53,596 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T06:21:53,597 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T06:21:53,609 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T06:21:53,612 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/hadoop.log.dir/,AVAILABLE} 2024-12-03T06:21:53,613 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T06:21:53,792 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64418798{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/java.io.tmpdir/jetty-localhost-35895-hadoop-hdfs-3_4_1-tests_jar-_-any-15637865728726412259/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T06:21:53,813 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:35895} 2024-12-03T06:21:53,813 INFO [Time-limited test {}] server.Server(415): Started @2891ms 2024-12-03T06:21:54,347 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T06:21:54,353 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T06:21:54,355 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T06:21:54,355 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T06:21:54,355 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T06:21:54,356 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/hadoop.log.dir/,AVAILABLE} 2024-12-03T06:21:54,357 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T06:21:54,452 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@79b30fb0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/java.io.tmpdir/jetty-localhost-46395-hadoop-hdfs-3_4_1-tests_jar-_-any-9515461984442831825/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T06:21:54,453 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:46395} 2024-12-03T06:21:54,453 INFO [Time-limited test {}] server.Server(415): Started @3531ms 2024-12-03T06:21:54,502 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T06:21:54,605 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T06:21:54,609 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T06:21:54,611 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T06:21:54,611 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T06:21:54,612 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T06:21:54,619 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/hadoop.log.dir/,AVAILABLE} 2024-12-03T06:21:54,620 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T06:21:54,727 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@34fcb585{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/java.io.tmpdir/jetty-localhost-44961-hadoop-hdfs-3_4_1-tests_jar-_-any-11373143056849684614/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T06:21:54,728 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:44961} 2024-12-03T06:21:54,728 INFO [Time-limited test {}] server.Server(415): Started @3806ms 2024-12-03T06:21:54,731 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T06:21:54,766 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T06:21:54,772 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T06:21:54,777 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T06:21:54,777 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T06:21:54,777 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T06:21:54,778 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/hadoop.log.dir/,AVAILABLE} 2024-12-03T06:21:54,778 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T06:21:54,872 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3c1f701{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/java.io.tmpdir/jetty-localhost-43097-hadoop-hdfs-3_4_1-tests_jar-_-any-6764073996724000581/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T06:21:54,873 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:43097} 2024-12-03T06:21:54,873 INFO [Time-limited test {}] server.Server(415): Started @3950ms 2024-12-03T06:21:54,875 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T06:21:55,663 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/cluster_9b3a36d6-da9f-80f6-d98c-a615b634f695/data/data2/current/BP-141962458-172.17.0.2-1733206912875/current, will proceed with Du for space computation calculation, 2024-12-03T06:21:55,663 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/cluster_9b3a36d6-da9f-80f6-d98c-a615b634f695/data/data4/current/BP-141962458-172.17.0.2-1733206912875/current, will proceed with Du for space computation calculation, 2024-12-03T06:21:55,663 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/cluster_9b3a36d6-da9f-80f6-d98c-a615b634f695/data/data3/current/BP-141962458-172.17.0.2-1733206912875/current, will proceed with Du for space computation calculation, 2024-12-03T06:21:55,663 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/cluster_9b3a36d6-da9f-80f6-d98c-a615b634f695/data/data1/current/BP-141962458-172.17.0.2-1733206912875/current, will proceed with Du for space computation calculation, 2024-12-03T06:21:55,697 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T06:21:55,698 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T06:21:55,727 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/cluster_9b3a36d6-da9f-80f6-d98c-a615b634f695/data/data5/current/BP-141962458-172.17.0.2-1733206912875/current, will proceed with Du for space computation calculation, 2024-12-03T06:21:55,727 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/cluster_9b3a36d6-da9f-80f6-d98c-a615b634f695/data/data6/current/BP-141962458-172.17.0.2-1733206912875/current, will proceed with Du for space computation calculation, 2024-12-03T06:21:55,745 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x25a5972d33cad4d1 with lease ID 0x21ec0706bda35678: Processing first storage report for DS-e82fae88-53d6-4925-8312-e8210040c0a7 from datanode DatanodeRegistration(127.0.0.1:40623, datanodeUuid=59069bac-f601-4e30-b0d8-e908363ae328, infoPort=39137, infoSecurePort=0, ipcPort=40125, storageInfo=lv=-57;cid=testClusterID;nsid=668991797;c=1733206912875) 2024-12-03T06:21:55,747 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x25a5972d33cad4d1 with lease ID 0x21ec0706bda35678: from storage DS-e82fae88-53d6-4925-8312-e8210040c0a7 node DatanodeRegistration(127.0.0.1:40623, datanodeUuid=59069bac-f601-4e30-b0d8-e908363ae328, infoPort=39137, infoSecurePort=0, ipcPort=40125, storageInfo=lv=-57;cid=testClusterID;nsid=668991797;c=1733206912875), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-03T06:21:55,747 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8a650c1478e0a1d9 with lease ID 0x21ec0706bda35679: Processing first storage report for DS-13cfb8bd-8a77-4bfd-914d-f7a5e73d8552 from datanode DatanodeRegistration(127.0.0.1:32933, datanodeUuid=1045b0d5-00e2-4e88-a577-6f1d9308a208, infoPort=36425, infoSecurePort=0, ipcPort=38397, storageInfo=lv=-57;cid=testClusterID;nsid=668991797;c=1733206912875) 2024-12-03T06:21:55,747 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T06:21:55,747 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8a650c1478e0a1d9 with lease ID 0x21ec0706bda35679: from storage DS-13cfb8bd-8a77-4bfd-914d-f7a5e73d8552 node DatanodeRegistration(127.0.0.1:32933, datanodeUuid=1045b0d5-00e2-4e88-a577-6f1d9308a208, infoPort=36425, infoSecurePort=0, ipcPort=38397, storageInfo=lv=-57;cid=testClusterID;nsid=668991797;c=1733206912875), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T06:21:55,748 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x25a5972d33cad4d1 with lease ID 0x21ec0706bda35678: Processing first storage report for DS-c249ea02-13b2-4387-b251-c9ea6563ea15 from datanode DatanodeRegistration(127.0.0.1:40623, datanodeUuid=59069bac-f601-4e30-b0d8-e908363ae328, infoPort=39137, infoSecurePort=0, ipcPort=40125, storageInfo=lv=-57;cid=testClusterID;nsid=668991797;c=1733206912875) 2024-12-03T06:21:55,748 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x25a5972d33cad4d1 with lease ID 0x21ec0706bda35678: from storage DS-c249ea02-13b2-4387-b251-c9ea6563ea15 node DatanodeRegistration(127.0.0.1:40623, datanodeUuid=59069bac-f601-4e30-b0d8-e908363ae328, infoPort=39137, infoSecurePort=0, ipcPort=40125, storageInfo=lv=-57;cid=testClusterID;nsid=668991797;c=1733206912875), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T06:21:55,748 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8a650c1478e0a1d9 with lease ID 0x21ec0706bda35679: Processing first storage report for DS-d23645e1-93ea-4dd0-82c9-7b115fccfcac from datanode DatanodeRegistration(127.0.0.1:32933, datanodeUuid=1045b0d5-00e2-4e88-a577-6f1d9308a208, infoPort=36425, infoSecurePort=0, ipcPort=38397, storageInfo=lv=-57;cid=testClusterID;nsid=668991797;c=1733206912875) 2024-12-03T06:21:55,748 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8a650c1478e0a1d9 with lease ID 0x21ec0706bda35679: from storage DS-d23645e1-93ea-4dd0-82c9-7b115fccfcac node DatanodeRegistration(127.0.0.1:32933, datanodeUuid=1045b0d5-00e2-4e88-a577-6f1d9308a208, infoPort=36425, infoSecurePort=0, ipcPort=38397, storageInfo=lv=-57;cid=testClusterID;nsid=668991797;c=1733206912875), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T06:21:55,753 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdd6378fcf7a682ee with lease ID 0x21ec0706bda3567a: Processing first storage report for DS-0c445287-33c1-4585-8daa-736a3c1373ca from datanode DatanodeRegistration(127.0.0.1:43645, datanodeUuid=90db776d-4a4a-401f-a7a8-677282c8abd6, infoPort=41213, infoSecurePort=0, ipcPort=33933, storageInfo=lv=-57;cid=testClusterID;nsid=668991797;c=1733206912875) 2024-12-03T06:21:55,753 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdd6378fcf7a682ee with lease ID 0x21ec0706bda3567a: from storage DS-0c445287-33c1-4585-8daa-736a3c1373ca node DatanodeRegistration(127.0.0.1:43645, datanodeUuid=90db776d-4a4a-401f-a7a8-677282c8abd6, infoPort=41213, infoSecurePort=0, ipcPort=33933, storageInfo=lv=-57;cid=testClusterID;nsid=668991797;c=1733206912875), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T06:21:55,753 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdd6378fcf7a682ee with lease ID 0x21ec0706bda3567a: Processing first storage report for DS-77548eec-3c60-48e7-8d82-70e964d2b3ac from datanode DatanodeRegistration(127.0.0.1:43645, datanodeUuid=90db776d-4a4a-401f-a7a8-677282c8abd6, infoPort=41213, infoSecurePort=0, ipcPort=33933, storageInfo=lv=-57;cid=testClusterID;nsid=668991797;c=1733206912875) 2024-12-03T06:21:55,753 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdd6378fcf7a682ee with lease ID 0x21ec0706bda3567a: from storage DS-77548eec-3c60-48e7-8d82-70e964d2b3ac node DatanodeRegistration(127.0.0.1:43645, datanodeUuid=90db776d-4a4a-401f-a7a8-677282c8abd6, infoPort=41213, infoSecurePort=0, ipcPort=33933, storageInfo=lv=-57;cid=testClusterID;nsid=668991797;c=1733206912875), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T06:21:55,804 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03 2024-12-03T06:21:55,876 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/cluster_9b3a36d6-da9f-80f6-d98c-a615b634f695/zookeeper_0, clientPort=63630, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/cluster_9b3a36d6-da9f-80f6-d98c-a615b634f695/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/cluster_9b3a36d6-da9f-80f6-d98c-a615b634f695/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-03T06:21:55,886 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63630 2024-12-03T06:21:55,897 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T06:21:55,901 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T06:21:56,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741825_1001 (size=7) 2024-12-03T06:21:56,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741825_1001 (size=7) 2024-12-03T06:21:56,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741825_1001 (size=7) 2024-12-03T06:21:56,526 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 with version=8 2024-12-03T06:21:56,526 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/hbase-staging 2024-12-03T06:21:56,602 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-03T06:21:56,797 INFO [Time-limited test {}] client.ConnectionUtils(128): master/122cddc95ec8:0 server-side Connection retries=45 2024-12-03T06:21:56,805 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T06:21:56,806 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T06:21:56,811 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T06:21:56,811 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T06:21:56,811 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T06:21:56,956 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-03T06:21:57,011 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-03T06:21:57,020 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-03T06:21:57,023 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T06:21:57,046 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 69922 (auto-detected) 2024-12-03T06:21:57,047 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-03T06:21:57,064 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35025 2024-12-03T06:21:57,085 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35025 connecting to ZooKeeper ensemble=127.0.0.1:63630 2024-12-03T06:21:57,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:350250x0, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T06:21:57,170 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35025-0x1009eb0b9ca0000 connected 2024-12-03T06:21:57,255 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T06:21:57,259 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T06:21:57,269 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T06:21:57,273 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261, hbase.cluster.distributed=false 2024-12-03T06:21:57,298 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T06:21:57,303 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35025 2024-12-03T06:21:57,304 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35025 2024-12-03T06:21:57,304 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35025 2024-12-03T06:21:57,305 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35025 2024-12-03T06:21:57,305 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35025 2024-12-03T06:21:57,399 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/122cddc95ec8:0 server-side Connection retries=45 2024-12-03T06:21:57,400 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T06:21:57,400 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T06:21:57,401 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T06:21:57,401 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T06:21:57,401 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T06:21:57,404 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T06:21:57,406 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T06:21:57,407 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43629 2024-12-03T06:21:57,408 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43629 connecting to ZooKeeper ensemble=127.0.0.1:63630 2024-12-03T06:21:57,409 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T06:21:57,411 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T06:21:57,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:436290x0, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T06:21:57,425 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43629-0x1009eb0b9ca0001 connected 2024-12-03T06:21:57,425 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T06:21:57,429 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T06:21:57,438 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T06:21:57,441 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T06:21:57,447 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T06:21:57,448 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43629 2024-12-03T06:21:57,448 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43629 2024-12-03T06:21:57,449 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43629 2024-12-03T06:21:57,449 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43629 2024-12-03T06:21:57,449 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43629 2024-12-03T06:21:57,466 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/122cddc95ec8:0 server-side Connection retries=45 2024-12-03T06:21:57,466 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T06:21:57,467 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T06:21:57,467 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T06:21:57,467 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T06:21:57,467 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T06:21:57,467 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T06:21:57,468 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T06:21:57,469 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45577 2024-12-03T06:21:57,470 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45577 connecting to ZooKeeper ensemble=127.0.0.1:63630 2024-12-03T06:21:57,471 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T06:21:57,473 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T06:21:57,486 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:455770x0, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T06:21:57,487 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45577-0x1009eb0b9ca0002 connected 2024-12-03T06:21:57,487 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T06:21:57,487 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T06:21:57,489 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T06:21:57,490 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T06:21:57,492 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T06:21:57,493 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45577 2024-12-03T06:21:57,493 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45577 2024-12-03T06:21:57,494 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45577 2024-12-03T06:21:57,494 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45577 2024-12-03T06:21:57,495 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45577 2024-12-03T06:21:57,509 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/122cddc95ec8:0 server-side Connection retries=45 2024-12-03T06:21:57,509 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T06:21:57,509 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T06:21:57,510 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T06:21:57,510 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T06:21:57,510 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T06:21:57,510 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T06:21:57,510 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T06:21:57,511 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37369 2024-12-03T06:21:57,512 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37369 connecting to ZooKeeper ensemble=127.0.0.1:63630 2024-12-03T06:21:57,513 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T06:21:57,516 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T06:21:57,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:373690x0, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T06:21:57,528 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:373690x0, quorum=127.0.0.1:63630, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T06:21:57,529 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37369-0x1009eb0b9ca0003 connected 2024-12-03T06:21:57,529 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T06:21:57,531 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T06:21:57,532 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37369-0x1009eb0b9ca0003, quorum=127.0.0.1:63630, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T06:21:57,534 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37369-0x1009eb0b9ca0003, quorum=127.0.0.1:63630, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T06:21:57,538 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37369 2024-12-03T06:21:57,538 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37369 2024-12-03T06:21:57,539 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37369 2024-12-03T06:21:57,540 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37369 2024-12-03T06:21:57,541 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37369 2024-12-03T06:21:57,559 DEBUG [M:0;122cddc95ec8:35025 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;122cddc95ec8:35025 2024-12-03T06:21:57,560 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/122cddc95ec8,35025,1733206916647 2024-12-03T06:21:57,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T06:21:57,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T06:21:57,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37369-0x1009eb0b9ca0003, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T06:21:57,575 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T06:21:57,578 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/122cddc95ec8,35025,1733206916647 2024-12-03T06:21:57,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37369-0x1009eb0b9ca0003, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T06:21:57,611 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T06:21:57,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37369-0x1009eb0b9ca0003, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:21:57,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T06:21:57,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:21:57,612 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:21:57,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:21:57,614 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-03T06:21:57,616 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/122cddc95ec8,35025,1733206916647 from backup master directory 2024-12-03T06:21:57,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/122cddc95ec8,35025,1733206916647 2024-12-03T06:21:57,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T06:21:57,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37369-0x1009eb0b9ca0003, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T06:21:57,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T06:21:57,629 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T06:21:57,629 WARN [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T06:21:57,630 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=122cddc95ec8,35025,1733206916647 2024-12-03T06:21:57,632 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-03T06:21:57,635 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-03T06:21:57,702 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/hbase.id] with ID: c0c7c2db-4b33-4027-9625-bcb0de82bf43 2024-12-03T06:21:57,702 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/.tmp/hbase.id 2024-12-03T06:21:57,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741826_1002 (size=42) 2024-12-03T06:21:57,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741826_1002 (size=42) 2024-12-03T06:21:57,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741826_1002 (size=42) 2024-12-03T06:21:57,719 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/.tmp/hbase.id]:[hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/hbase.id] 2024-12-03T06:21:57,769 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T06:21:57,774 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-03T06:21:57,793 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-12-03T06:21:57,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37369-0x1009eb0b9ca0003, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:21:57,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:21:57,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:21:57,803 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:21:57,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741827_1003 (size=196) 2024-12-03T06:21:57,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741827_1003 (size=196) 2024-12-03T06:21:57,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741827_1003 (size=196) 2024-12-03T06:21:57,837 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T06:21:57,838 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-03T06:21:57,852 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:21:57,856 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T06:21:57,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741828_1004 (size=1189) 2024-12-03T06:21:57,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741828_1004 (size=1189) 2024-12-03T06:21:57,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741828_1004 (size=1189) 2024-12-03T06:21:57,904 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/data/master/store 2024-12-03T06:21:57,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741829_1005 (size=34) 2024-12-03T06:21:57,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741829_1005 (size=34) 2024-12-03T06:21:57,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741829_1005 (size=34) 2024-12-03T06:21:57,931 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-03T06:21:57,935 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:21:57,937 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T06:21:57,937 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T06:21:57,937 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T06:21:57,939 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T06:21:57,940 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T06:21:57,940 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T06:21:57,942 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733206917936Disabling compacts and flushes for region at 1733206917936Disabling writes for close at 1733206917940 (+4 ms)Writing region close event to WAL at 1733206917940Closed at 1733206917940 2024-12-03T06:21:57,944 WARN [master/122cddc95ec8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/data/master/store/.initializing 2024-12-03T06:21:57,944 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/WALs/122cddc95ec8,35025,1733206916647 2024-12-03T06:21:57,953 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T06:21:57,970 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=122cddc95ec8%2C35025%2C1733206916647, suffix=, logDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/WALs/122cddc95ec8,35025,1733206916647, archiveDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/oldWALs, maxLogs=10 2024-12-03T06:21:57,992 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/WALs/122cddc95ec8,35025,1733206916647/122cddc95ec8%2C35025%2C1733206916647.1733206917975, exclude list is [], retry=0 2024-12-03T06:21:58,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40623,DS-e82fae88-53d6-4925-8312-e8210040c0a7,DISK] 2024-12-03T06:21:58,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32933,DS-13cfb8bd-8a77-4bfd-914d-f7a5e73d8552,DISK] 2024-12-03T06:21:58,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43645,DS-0c445287-33c1-4585-8daa-736a3c1373ca,DISK] 2024-12-03T06:21:58,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-03T06:21:58,051 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/WALs/122cddc95ec8,35025,1733206916647/122cddc95ec8%2C35025%2C1733206916647.1733206917975 2024-12-03T06:21:58,052 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41213:41213),(127.0.0.1/127.0.0.1:36425:36425),(127.0.0.1/127.0.0.1:39137:39137)] 2024-12-03T06:21:58,053 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-03T06:21:58,053 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:21:58,056 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T06:21:58,057 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T06:21:58,093 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T06:21:58,119 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-03T06:21:58,123 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:21:58,125 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T06:21:58,126 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T06:21:58,129 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-03T06:21:58,130 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:21:58,131 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:21:58,131 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T06:21:58,134 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-03T06:21:58,135 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:21:58,136 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:21:58,136 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T06:21:58,139 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-03T06:21:58,139 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:21:58,140 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:21:58,141 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T06:21:58,144 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T06:21:58,145 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T06:21:58,151 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T06:21:58,152 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T06:21:58,155 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T06:21:58,159 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T06:21:58,163 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:21:58,165 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72616757, jitterRate=0.08207400143146515}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T06:21:58,170 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733206918069Initializing all the Stores at 1733206918072 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733206918073 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206918073Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206918074 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206918074Cleaning up temporary data from old regions at 1733206918152 (+78 ms)Region opened successfully at 1733206918170 (+18 ms) 2024-12-03T06:21:58,171 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-03T06:21:58,201 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35f11130, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=122cddc95ec8/172.17.0.2:0 2024-12-03T06:21:58,227 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-03T06:21:58,236 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T06:21:58,236 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T06:21:58,239 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T06:21:58,240 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-03T06:21:58,244 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-03T06:21:58,244 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-03T06:21:58,266 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-03T06:21:58,274 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-03T06:21:58,319 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-03T06:21:58,323 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-03T06:21:58,326 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-03T06:21:58,336 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-03T06:21:58,339 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-03T06:21:58,344 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-03T06:21:58,352 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-03T06:21:58,353 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-03T06:21:58,361 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-03T06:21:58,385 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-03T06:21:58,394 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-03T06:21:58,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T06:21:58,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37369-0x1009eb0b9ca0003, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T06:21:58,407 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T06:21:58,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T06:21:58,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:21:58,408 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:21:58,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:21:58,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37369-0x1009eb0b9ca0003, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:21:58,412 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=122cddc95ec8,35025,1733206916647, sessionid=0x1009eb0b9ca0000, setting cluster-up flag (Was=false) 2024-12-03T06:21:58,440 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:21:58,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:21:58,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37369-0x1009eb0b9ca0003, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:21:58,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:21:58,466 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-03T06:21:58,469 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=122cddc95ec8,35025,1733206916647 2024-12-03T06:21:58,491 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:21:58,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:21:58,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:21:58,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37369-0x1009eb0b9ca0003, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:21:58,519 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-03T06:21:58,523 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=122cddc95ec8,35025,1733206916647 2024-12-03T06:21:58,534 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-03T06:21:58,545 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.HRegionServer(746): ClusterId : c0c7c2db-4b33-4027-9625-bcb0de82bf43 2024-12-03T06:21:58,545 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.HRegionServer(746): ClusterId : c0c7c2db-4b33-4027-9625-bcb0de82bf43 2024-12-03T06:21:58,545 INFO [RS:2;122cddc95ec8:37369 {}] regionserver.HRegionServer(746): ClusterId : c0c7c2db-4b33-4027-9625-bcb0de82bf43 2024-12-03T06:21:58,548 DEBUG [RS:0;122cddc95ec8:43629 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T06:21:58,548 DEBUG [RS:1;122cddc95ec8:45577 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T06:21:58,548 DEBUG [RS:2;122cddc95ec8:37369 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T06:21:58,559 DEBUG [RS:1;122cddc95ec8:45577 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T06:21:58,559 DEBUG [RS:0;122cddc95ec8:43629 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T06:21:58,559 DEBUG [RS:2;122cddc95ec8:37369 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T06:21:58,559 DEBUG [RS:1;122cddc95ec8:45577 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T06:21:58,559 DEBUG [RS:0;122cddc95ec8:43629 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T06:21:58,559 DEBUG [RS:2;122cddc95ec8:37369 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T06:21:58,578 DEBUG [RS:1;122cddc95ec8:45577 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T06:21:58,578 DEBUG [RS:0;122cddc95ec8:43629 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T06:21:58,578 DEBUG [RS:2;122cddc95ec8:37369 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T06:21:58,579 DEBUG [RS:1;122cddc95ec8:45577 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d725e84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=122cddc95ec8/172.17.0.2:0 2024-12-03T06:21:58,579 DEBUG [RS:2;122cddc95ec8:37369 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@339db88b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=122cddc95ec8/172.17.0.2:0 2024-12-03T06:21:58,579 DEBUG [RS:0;122cddc95ec8:43629 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a1efdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=122cddc95ec8/172.17.0.2:0 2024-12-03T06:21:58,597 DEBUG [RS:1;122cddc95ec8:45577 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;122cddc95ec8:45577 2024-12-03T06:21:58,597 DEBUG [RS:2;122cddc95ec8:37369 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;122cddc95ec8:37369 2024-12-03T06:21:58,597 DEBUG [RS:0;122cddc95ec8:43629 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;122cddc95ec8:43629 2024-12-03T06:21:58,601 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T06:21:58,601 INFO [RS:2;122cddc95ec8:37369 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T06:21:58,601 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T06:21:58,601 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T06:21:58,601 INFO [RS:2;122cddc95ec8:37369 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T06:21:58,601 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T06:21:58,601 DEBUG [RS:0;122cddc95ec8:43629 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T06:21:58,601 DEBUG [RS:1;122cddc95ec8:45577 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T06:21:58,601 DEBUG [RS:2;122cddc95ec8:37369 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T06:21:58,604 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.HRegionServer(2659): reportForDuty to master=122cddc95ec8,35025,1733206916647 with port=43629, startcode=1733206917367 2024-12-03T06:21:58,604 INFO [RS:2;122cddc95ec8:37369 {}] regionserver.HRegionServer(2659): reportForDuty to master=122cddc95ec8,35025,1733206916647 with port=37369, startcode=1733206917509 2024-12-03T06:21:58,604 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.HRegionServer(2659): reportForDuty to master=122cddc95ec8,35025,1733206916647 with port=45577, startcode=1733206917466 2024-12-03T06:21:58,612 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T06:21:58,617 DEBUG [RS:2;122cddc95ec8:37369 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T06:21:58,617 DEBUG [RS:0;122cddc95ec8:43629 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T06:21:58,617 DEBUG [RS:1;122cddc95ec8:45577 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T06:21:58,621 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-03T06:21:58,629 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-03T06:21:58,635 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 122cddc95ec8,35025,1733206916647 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-03T06:21:58,642 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/122cddc95ec8:0, corePoolSize=5, maxPoolSize=5 2024-12-03T06:21:58,643 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/122cddc95ec8:0, corePoolSize=5, maxPoolSize=5 2024-12-03T06:21:58,644 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/122cddc95ec8:0, corePoolSize=5, maxPoolSize=5 2024-12-03T06:21:58,644 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/122cddc95ec8:0, corePoolSize=5, maxPoolSize=5 2024-12-03T06:21:58,645 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/122cddc95ec8:0, corePoolSize=10, maxPoolSize=10 2024-12-03T06:21:58,645 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,645 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/122cddc95ec8:0, corePoolSize=2, maxPoolSize=2 2024-12-03T06:21:58,645 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,646 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733206948646 2024-12-03T06:21:58,648 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-03T06:21:58,649 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-03T06:21:58,651 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T06:21:58,652 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-03T06:21:58,656 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-03T06:21:58,656 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34345, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T06:21:58,656 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37585, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T06:21:58,656 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43079, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T06:21:58,656 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-03T06:21:58,657 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-03T06:21:58,657 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-03T06:21:58,658 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:21:58,659 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-03T06:21:58,658 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,663 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-03T06:21:58,663 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:21:58,664 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-03T06:21:58,665 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-03T06:21:58,667 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-03T06:21:58,667 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-03T06:21:58,669 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:21:58,670 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:21:58,674 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/122cddc95ec8:0:becomeActiveMaster-HFileCleaner.large.0-1733206918669,5,FailOnTimeoutGroup] 2024-12-03T06:21:58,678 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/122cddc95ec8:0:becomeActiveMaster-HFileCleaner.small.0-1733206918674,5,FailOnTimeoutGroup] 2024-12-03T06:21:58,678 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,678 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-03T06:21:58,707 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,708 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741831_1007 (size=1321) 2024-12-03T06:21:58,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741831_1007 (size=1321) 2024-12-03T06:21:58,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741831_1007 (size=1321) 2024-12-03T06:21:58,712 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-03T06:21:58,712 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:21:58,722 DEBUG [RS:2;122cddc95ec8:37369 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-03T06:21:58,722 DEBUG [RS:0;122cddc95ec8:43629 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-03T06:21:58,722 DEBUG [RS:1;122cddc95ec8:45577 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-03T06:21:58,722 WARN [RS:1;122cddc95ec8:45577 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-03T06:21:58,722 WARN [RS:2;122cddc95ec8:37369 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-03T06:21:58,722 WARN [RS:0;122cddc95ec8:43629 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-03T06:21:58,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741832_1008 (size=32) 2024-12-03T06:21:58,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741832_1008 (size=32) 2024-12-03T06:21:58,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741832_1008 (size=32) 2024-12-03T06:21:58,732 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:21:58,734 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T06:21:58,736 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T06:21:58,736 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:21:58,737 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T06:21:58,737 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T06:21:58,740 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T06:21:58,740 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:21:58,741 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T06:21:58,741 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T06:21:58,744 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T06:21:58,744 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:21:58,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T06:21:58,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T06:21:58,748 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T06:21:58,749 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:21:58,750 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T06:21:58,750 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T06:21:58,752 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740 2024-12-03T06:21:58,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740 2024-12-03T06:21:58,756 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T06:21:58,757 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T06:21:58,758 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T06:21:58,760 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T06:21:58,765 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:21:58,766 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73923126, jitterRate=0.10154041647911072}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T06:21:58,769 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733206918732Initializing all the Stores at 1733206918733 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733206918733Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733206918733Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206918733Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733206918733Cleaning up temporary data from old regions at 1733206918757 (+24 ms)Region opened successfully at 1733206918769 (+12 ms) 2024-12-03T06:21:58,769 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T06:21:58,769 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T06:21:58,770 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T06:21:58,770 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T06:21:58,770 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T06:21:58,771 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T06:21:58,771 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733206918769Disabling compacts and flushes for region at 1733206918769Disabling writes for close at 1733206918770 (+1 ms)Writing region close event to WAL at 1733206918771 (+1 ms)Closed at 1733206918771 2024-12-03T06:21:58,775 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T06:21:58,775 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-03T06:21:58,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-03T06:21:58,791 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T06:21:58,794 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-03T06:21:58,823 INFO [RS:2;122cddc95ec8:37369 {}] regionserver.HRegionServer(2659): reportForDuty to master=122cddc95ec8,35025,1733206916647 with port=37369, startcode=1733206917509 2024-12-03T06:21:58,823 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.HRegionServer(2659): reportForDuty to master=122cddc95ec8,35025,1733206916647 with port=43629, startcode=1733206917367 2024-12-03T06:21:58,823 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.HRegionServer(2659): reportForDuty to master=122cddc95ec8,35025,1733206916647 with port=45577, startcode=1733206917466 2024-12-03T06:21:58,826 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 122cddc95ec8,45577,1733206917466 2024-12-03T06:21:58,828 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] master.ServerManager(517): Registering regionserver=122cddc95ec8,45577,1733206917466 2024-12-03T06:21:58,835 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 122cddc95ec8,43629,1733206917367 2024-12-03T06:21:58,835 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] master.ServerManager(517): Registering regionserver=122cddc95ec8,43629,1733206917367 2024-12-03T06:21:58,835 DEBUG [RS:1;122cddc95ec8:45577 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:21:58,835 DEBUG [RS:1;122cddc95ec8:45577 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34349 2024-12-03T06:21:58,836 DEBUG [RS:1;122cddc95ec8:45577 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T06:21:58,838 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 122cddc95ec8,37369,1733206917509 2024-12-03T06:21:58,838 DEBUG [RS:0;122cddc95ec8:43629 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:21:58,838 DEBUG [RS:0;122cddc95ec8:43629 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34349 2024-12-03T06:21:58,838 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] master.ServerManager(517): Registering regionserver=122cddc95ec8,37369,1733206917509 2024-12-03T06:21:58,838 DEBUG [RS:0;122cddc95ec8:43629 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T06:21:58,841 DEBUG [RS:2;122cddc95ec8:37369 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:21:58,841 DEBUG [RS:2;122cddc95ec8:37369 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34349 2024-12-03T06:21:58,841 DEBUG [RS:2;122cddc95ec8:37369 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T06:21:58,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T06:21:58,881 DEBUG [RS:1;122cddc95ec8:45577 {}] zookeeper.ZKUtil(111): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/122cddc95ec8,45577,1733206917466 2024-12-03T06:21:58,881 WARN [RS:1;122cddc95ec8:45577 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T06:21:58,882 INFO [RS:1;122cddc95ec8:45577 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T06:21:58,882 DEBUG [RS:0;122cddc95ec8:43629 {}] zookeeper.ZKUtil(111): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/122cddc95ec8,43629,1733206917367 2024-12-03T06:21:58,882 DEBUG [RS:2;122cddc95ec8:37369 {}] zookeeper.ZKUtil(111): regionserver:37369-0x1009eb0b9ca0003, quorum=127.0.0.1:63630, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/122cddc95ec8,37369,1733206917509 2024-12-03T06:21:58,882 WARN [RS:0;122cddc95ec8:43629 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T06:21:58,882 WARN [RS:2;122cddc95ec8:37369 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T06:21:58,882 INFO [RS:0;122cddc95ec8:43629 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T06:21:58,882 INFO [RS:2;122cddc95ec8:37369 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T06:21:58,882 DEBUG [RS:1;122cddc95ec8:45577 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,45577,1733206917466 2024-12-03T06:21:58,883 DEBUG [RS:2;122cddc95ec8:37369 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509 2024-12-03T06:21:58,883 DEBUG [RS:0;122cddc95ec8:43629 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,43629,1733206917367 2024-12-03T06:21:58,885 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [122cddc95ec8,43629,1733206917367] 2024-12-03T06:21:58,885 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [122cddc95ec8,45577,1733206917466] 2024-12-03T06:21:58,885 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [122cddc95ec8,37369,1733206917509] 2024-12-03T06:21:58,911 INFO [RS:2;122cddc95ec8:37369 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T06:21:58,911 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T06:21:58,911 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T06:21:58,925 INFO [RS:2;122cddc95ec8:37369 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T06:21:58,925 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T06:21:58,925 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T06:21:58,931 INFO [RS:0;122cddc95ec8:43629 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T06:21:58,931 INFO [RS:1;122cddc95ec8:45577 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T06:21:58,931 INFO [RS:2;122cddc95ec8:37369 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T06:21:58,931 INFO [RS:0;122cddc95ec8:43629 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,931 INFO [RS:1;122cddc95ec8:45577 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,931 INFO [RS:2;122cddc95ec8:37369 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,932 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T06:21:58,932 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T06:21:58,932 INFO [RS:2;122cddc95ec8:37369 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T06:21:58,937 INFO [RS:1;122cddc95ec8:45577 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T06:21:58,937 INFO [RS:0;122cddc95ec8:43629 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T06:21:58,937 INFO [RS:2;122cddc95ec8:37369 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T06:21:58,938 INFO [RS:1;122cddc95ec8:45577 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,938 INFO [RS:2;122cddc95ec8:37369 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,938 INFO [RS:0;122cddc95ec8:43629 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,939 DEBUG [RS:1;122cddc95ec8:45577 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,939 DEBUG [RS:2;122cddc95ec8:37369 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,939 DEBUG [RS:0;122cddc95ec8:43629 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,939 DEBUG [RS:1;122cddc95ec8:45577 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,939 DEBUG [RS:2;122cddc95ec8:37369 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,939 DEBUG [RS:0;122cddc95ec8:43629 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,939 DEBUG [RS:1;122cddc95ec8:45577 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,939 DEBUG [RS:0;122cddc95ec8:43629 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,939 DEBUG [RS:2;122cddc95ec8:37369 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,939 DEBUG [RS:1;122cddc95ec8:45577 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,939 DEBUG [RS:0;122cddc95ec8:43629 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,939 DEBUG [RS:1;122cddc95ec8:45577 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,939 DEBUG [RS:2;122cddc95ec8:37369 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,939 DEBUG [RS:0;122cddc95ec8:43629 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,939 DEBUG [RS:1;122cddc95ec8:45577 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0, corePoolSize=2, maxPoolSize=2 2024-12-03T06:21:58,939 DEBUG [RS:2;122cddc95ec8:37369 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,940 DEBUG [RS:0;122cddc95ec8:43629 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0, corePoolSize=2, maxPoolSize=2 2024-12-03T06:21:58,940 DEBUG [RS:1;122cddc95ec8:45577 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,940 DEBUG [RS:2;122cddc95ec8:37369 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0, corePoolSize=2, maxPoolSize=2 2024-12-03T06:21:58,940 DEBUG [RS:0;122cddc95ec8:43629 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,940 DEBUG [RS:1;122cddc95ec8:45577 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,940 DEBUG [RS:2;122cddc95ec8:37369 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,940 DEBUG [RS:0;122cddc95ec8:43629 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,940 DEBUG [RS:2;122cddc95ec8:37369 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,940 DEBUG [RS:1;122cddc95ec8:45577 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,940 DEBUG [RS:0;122cddc95ec8:43629 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,940 DEBUG [RS:2;122cddc95ec8:37369 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,940 DEBUG [RS:1;122cddc95ec8:45577 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,940 DEBUG [RS:0;122cddc95ec8:43629 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,940 DEBUG [RS:2;122cddc95ec8:37369 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,940 DEBUG [RS:1;122cddc95ec8:45577 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,940 DEBUG [RS:0;122cddc95ec8:43629 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,940 DEBUG [RS:2;122cddc95ec8:37369 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,940 DEBUG [RS:1;122cddc95ec8:45577 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,940 DEBUG [RS:0;122cddc95ec8:43629 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,941 DEBUG [RS:2;122cddc95ec8:37369 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/122cddc95ec8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T06:21:58,941 DEBUG [RS:1;122cddc95ec8:45577 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/122cddc95ec8:0, corePoolSize=3, maxPoolSize=3 2024-12-03T06:21:58,941 DEBUG [RS:0;122cddc95ec8:43629 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/122cddc95ec8:0, corePoolSize=3, maxPoolSize=3 2024-12-03T06:21:58,941 DEBUG [RS:2;122cddc95ec8:37369 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/122cddc95ec8:0, corePoolSize=3, maxPoolSize=3 2024-12-03T06:21:58,941 DEBUG [RS:0;122cddc95ec8:43629 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0, corePoolSize=3, maxPoolSize=3 2024-12-03T06:21:58,941 DEBUG [RS:1;122cddc95ec8:45577 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0, corePoolSize=3, maxPoolSize=3 2024-12-03T06:21:58,941 DEBUG [RS:2;122cddc95ec8:37369 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/122cddc95ec8:0, corePoolSize=3, maxPoolSize=3 2024-12-03T06:21:58,943 INFO [RS:1;122cddc95ec8:45577 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,943 INFO [RS:1;122cddc95ec8:45577 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,943 INFO [RS:1;122cddc95ec8:45577 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,943 INFO [RS:1;122cddc95ec8:45577 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,943 INFO [RS:1;122cddc95ec8:45577 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,943 INFO [RS:1;122cddc95ec8:45577 {}] hbase.ChoreService(168): Chore ScheduledChore name=122cddc95ec8,45577,1733206917466-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T06:21:58,944 INFO [RS:2;122cddc95ec8:37369 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,944 INFO [RS:2;122cddc95ec8:37369 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,944 INFO [RS:0;122cddc95ec8:43629 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,944 INFO [RS:2;122cddc95ec8:37369 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,944 INFO [RS:0;122cddc95ec8:43629 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,945 INFO [RS:0;122cddc95ec8:43629 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,945 INFO [RS:2;122cddc95ec8:37369 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,945 INFO [RS:0;122cddc95ec8:43629 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,945 INFO [RS:0;122cddc95ec8:43629 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,945 INFO [RS:2;122cddc95ec8:37369 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,945 INFO [RS:0;122cddc95ec8:43629 {}] hbase.ChoreService(168): Chore ScheduledChore name=122cddc95ec8,43629,1733206917367-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T06:21:58,945 INFO [RS:2;122cddc95ec8:37369 {}] hbase.ChoreService(168): Chore ScheduledChore name=122cddc95ec8,37369,1733206917509-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T06:21:58,945 WARN [122cddc95ec8:35025 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T06:21:58,962 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T06:21:58,964 INFO [RS:0;122cddc95ec8:43629 {}] hbase.ChoreService(168): Chore ScheduledChore name=122cddc95ec8,43629,1733206917367-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,964 INFO [RS:0;122cddc95ec8:43629 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,964 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.Replication(171): 122cddc95ec8,43629,1733206917367 started 2024-12-03T06:21:58,965 INFO [RS:2;122cddc95ec8:37369 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T06:21:58,965 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T06:21:58,965 INFO [RS:2;122cddc95ec8:37369 {}] hbase.ChoreService(168): Chore ScheduledChore name=122cddc95ec8,37369,1733206917509-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,965 INFO [RS:1;122cddc95ec8:45577 {}] hbase.ChoreService(168): Chore ScheduledChore name=122cddc95ec8,45577,1733206917466-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,965 INFO [RS:1;122cddc95ec8:45577 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,965 INFO [RS:2;122cddc95ec8:37369 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,965 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.Replication(171): 122cddc95ec8,45577,1733206917466 started 2024-12-03T06:21:58,965 INFO [RS:2;122cddc95ec8:37369 {}] regionserver.Replication(171): 122cddc95ec8,37369,1733206917509 started 2024-12-03T06:21:58,981 INFO [RS:0;122cddc95ec8:43629 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,982 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.HRegionServer(1482): Serving as 122cddc95ec8,43629,1733206917367, RpcServer on 122cddc95ec8/172.17.0.2:43629, sessionid=0x1009eb0b9ca0001 2024-12-03T06:21:58,982 DEBUG [RS:0;122cddc95ec8:43629 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T06:21:58,982 DEBUG [RS:0;122cddc95ec8:43629 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 122cddc95ec8,43629,1733206917367 2024-12-03T06:21:58,983 DEBUG [RS:0;122cddc95ec8:43629 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '122cddc95ec8,43629,1733206917367' 2024-12-03T06:21:58,983 DEBUG [RS:0;122cddc95ec8:43629 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T06:21:58,984 DEBUG [RS:0;122cddc95ec8:43629 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T06:21:58,984 DEBUG [RS:0;122cddc95ec8:43629 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T06:21:58,984 DEBUG [RS:0;122cddc95ec8:43629 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T06:21:58,985 DEBUG [RS:0;122cddc95ec8:43629 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 122cddc95ec8,43629,1733206917367 2024-12-03T06:21:58,985 DEBUG [RS:0;122cddc95ec8:43629 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '122cddc95ec8,43629,1733206917367' 2024-12-03T06:21:58,985 DEBUG [RS:0;122cddc95ec8:43629 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T06:21:58,985 DEBUG [RS:0;122cddc95ec8:43629 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T06:21:58,986 DEBUG [RS:0;122cddc95ec8:43629 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T06:21:58,986 INFO [RS:0;122cddc95ec8:43629 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T06:21:58,986 INFO [RS:0;122cddc95ec8:43629 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T06:21:58,986 INFO [RS:1;122cddc95ec8:45577 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,986 INFO [RS:2;122cddc95ec8:37369 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:58,986 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.HRegionServer(1482): Serving as 122cddc95ec8,45577,1733206917466, RpcServer on 122cddc95ec8/172.17.0.2:45577, sessionid=0x1009eb0b9ca0002 2024-12-03T06:21:58,986 INFO [RS:2;122cddc95ec8:37369 {}] regionserver.HRegionServer(1482): Serving as 122cddc95ec8,37369,1733206917509, RpcServer on 122cddc95ec8/172.17.0.2:37369, sessionid=0x1009eb0b9ca0003 2024-12-03T06:21:58,987 DEBUG [RS:1;122cddc95ec8:45577 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T06:21:58,987 DEBUG [RS:2;122cddc95ec8:37369 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T06:21:58,987 DEBUG [RS:2;122cddc95ec8:37369 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 122cddc95ec8,37369,1733206917509 2024-12-03T06:21:58,987 DEBUG [RS:1;122cddc95ec8:45577 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 122cddc95ec8,45577,1733206917466 2024-12-03T06:21:58,987 DEBUG [RS:2;122cddc95ec8:37369 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '122cddc95ec8,37369,1733206917509' 2024-12-03T06:21:58,987 DEBUG [RS:1;122cddc95ec8:45577 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '122cddc95ec8,45577,1733206917466' 2024-12-03T06:21:58,987 DEBUG [RS:2;122cddc95ec8:37369 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T06:21:58,987 DEBUG [RS:1;122cddc95ec8:45577 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T06:21:58,988 DEBUG [RS:1;122cddc95ec8:45577 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T06:21:58,988 DEBUG [RS:2;122cddc95ec8:37369 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T06:21:58,988 DEBUG [RS:1;122cddc95ec8:45577 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T06:21:58,988 DEBUG [RS:2;122cddc95ec8:37369 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T06:21:58,988 DEBUG [RS:1;122cddc95ec8:45577 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T06:21:58,988 DEBUG [RS:2;122cddc95ec8:37369 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T06:21:58,988 DEBUG [RS:1;122cddc95ec8:45577 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 122cddc95ec8,45577,1733206917466 2024-12-03T06:21:58,988 DEBUG [RS:2;122cddc95ec8:37369 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 122cddc95ec8,37369,1733206917509 2024-12-03T06:21:58,988 DEBUG [RS:1;122cddc95ec8:45577 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '122cddc95ec8,45577,1733206917466' 2024-12-03T06:21:58,988 DEBUG [RS:2;122cddc95ec8:37369 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '122cddc95ec8,37369,1733206917509' 2024-12-03T06:21:58,988 DEBUG [RS:1;122cddc95ec8:45577 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T06:21:58,988 DEBUG [RS:2;122cddc95ec8:37369 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T06:21:58,989 DEBUG [RS:1;122cddc95ec8:45577 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T06:21:58,989 DEBUG [RS:2;122cddc95ec8:37369 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T06:21:58,989 DEBUG [RS:1;122cddc95ec8:45577 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T06:21:58,989 DEBUG [RS:2;122cddc95ec8:37369 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T06:21:58,989 INFO [RS:1;122cddc95ec8:45577 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T06:21:58,990 INFO [RS:2;122cddc95ec8:37369 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T06:21:58,990 INFO [RS:1;122cddc95ec8:45577 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T06:21:58,990 INFO [RS:2;122cddc95ec8:37369 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T06:21:59,092 INFO [RS:2;122cddc95ec8:37369 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T06:21:59,092 INFO [RS:0;122cddc95ec8:43629 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T06:21:59,092 INFO [RS:1;122cddc95ec8:45577 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T06:21:59,096 INFO [RS:0;122cddc95ec8:43629 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=122cddc95ec8%2C43629%2C1733206917367, suffix=, logDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,43629,1733206917367, archiveDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/oldWALs, maxLogs=32 2024-12-03T06:21:59,096 INFO [RS:2;122cddc95ec8:37369 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=122cddc95ec8%2C37369%2C1733206917509, suffix=, logDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509, archiveDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/oldWALs, maxLogs=32 2024-12-03T06:21:59,096 INFO [RS:1;122cddc95ec8:45577 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=122cddc95ec8%2C45577%2C1733206917466, suffix=, logDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,45577,1733206917466, archiveDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/oldWALs, maxLogs=32 2024-12-03T06:21:59,112 DEBUG [RS:0;122cddc95ec8:43629 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,43629,1733206917367/122cddc95ec8%2C43629%2C1733206917367.1733206919099, exclude list is [], retry=0 2024-12-03T06:21:59,117 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43645,DS-0c445287-33c1-4585-8daa-736a3c1373ca,DISK] 2024-12-03T06:21:59,117 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32933,DS-13cfb8bd-8a77-4bfd-914d-f7a5e73d8552,DISK] 2024-12-03T06:21:59,117 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40623,DS-e82fae88-53d6-4925-8312-e8210040c0a7,DISK] 2024-12-03T06:21:59,120 DEBUG [RS:2;122cddc95ec8:37369 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509/122cddc95ec8%2C37369%2C1733206917509.1733206919100, exclude list is [], retry=0 2024-12-03T06:21:59,120 DEBUG [RS:1;122cddc95ec8:45577 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,45577,1733206917466/122cddc95ec8%2C45577%2C1733206917466.1733206919100, exclude list is [], retry=0 2024-12-03T06:21:59,124 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40623,DS-e82fae88-53d6-4925-8312-e8210040c0a7,DISK] 2024-12-03T06:21:59,124 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43645,DS-0c445287-33c1-4585-8daa-736a3c1373ca,DISK] 2024-12-03T06:21:59,124 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43645,DS-0c445287-33c1-4585-8daa-736a3c1373ca,DISK] 2024-12-03T06:21:59,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32933,DS-13cfb8bd-8a77-4bfd-914d-f7a5e73d8552,DISK] 2024-12-03T06:21:59,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40623,DS-e82fae88-53d6-4925-8312-e8210040c0a7,DISK] 2024-12-03T06:21:59,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32933,DS-13cfb8bd-8a77-4bfd-914d-f7a5e73d8552,DISK] 2024-12-03T06:21:59,127 INFO [RS:0;122cddc95ec8:43629 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,43629,1733206917367/122cddc95ec8%2C43629%2C1733206917367.1733206919099 2024-12-03T06:21:59,128 DEBUG [RS:0;122cddc95ec8:43629 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41213:41213),(127.0.0.1/127.0.0.1:36425:36425),(127.0.0.1/127.0.0.1:39137:39137)] 2024-12-03T06:21:59,129 INFO [RS:2;122cddc95ec8:37369 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509/122cddc95ec8%2C37369%2C1733206917509.1733206919100 2024-12-03T06:21:59,130 INFO [RS:1;122cddc95ec8:45577 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,45577,1733206917466/122cddc95ec8%2C45577%2C1733206917466.1733206919100 2024-12-03T06:21:59,132 DEBUG [RS:2;122cddc95ec8:37369 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39137:39137),(127.0.0.1/127.0.0.1:36425:36425),(127.0.0.1/127.0.0.1:41213:41213)] 2024-12-03T06:21:59,132 DEBUG [RS:1;122cddc95ec8:45577 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41213:41213),(127.0.0.1/127.0.0.1:39137:39137),(127.0.0.1/127.0.0.1:36425:36425)] 2024-12-03T06:21:59,198 DEBUG [122cddc95ec8:35025 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-03T06:21:59,208 DEBUG [122cddc95ec8:35025 {}] balancer.BalancerClusterState(204): Hosts are {122cddc95ec8=0} racks are {/default-rack=0} 2024-12-03T06:21:59,215 DEBUG [122cddc95ec8:35025 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T06:21:59,215 DEBUG [122cddc95ec8:35025 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T06:21:59,215 DEBUG [122cddc95ec8:35025 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-03T06:21:59,215 DEBUG [122cddc95ec8:35025 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T06:21:59,215 DEBUG [122cddc95ec8:35025 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T06:21:59,215 DEBUG [122cddc95ec8:35025 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-03T06:21:59,215 INFO [122cddc95ec8:35025 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T06:21:59,215 INFO [122cddc95ec8:35025 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T06:21:59,215 INFO [122cddc95ec8:35025 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-03T06:21:59,215 DEBUG [122cddc95ec8:35025 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T06:21:59,221 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=122cddc95ec8,37369,1733206917509 2024-12-03T06:21:59,228 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 122cddc95ec8,37369,1733206917509, state=OPENING 2024-12-03T06:21:59,269 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-03T06:21:59,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37369-0x1009eb0b9ca0003, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:21:59,278 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:21:59,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:21:59,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:21:59,280 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:21:59,280 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:21:59,280 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:21:59,280 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:21:59,284 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T06:21:59,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,37369,1733206917509}] 2024-12-03T06:21:59,467 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T06:21:59,469 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37751, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T06:21:59,482 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-03T06:21:59,482 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T06:21:59,483 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-03T06:21:59,486 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=122cddc95ec8%2C37369%2C1733206917509.meta, suffix=.meta, logDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509, archiveDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/oldWALs, maxLogs=32 2024-12-03T06:21:59,502 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509/122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, exclude list is [], retry=0 2024-12-03T06:21:59,506 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40623,DS-e82fae88-53d6-4925-8312-e8210040c0a7,DISK] 2024-12-03T06:21:59,506 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32933,DS-13cfb8bd-8a77-4bfd-914d-f7a5e73d8552,DISK] 2024-12-03T06:21:59,506 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43645,DS-0c445287-33c1-4585-8daa-736a3c1373ca,DISK] 2024-12-03T06:21:59,510 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509/122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta 2024-12-03T06:21:59,511 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39137:39137),(127.0.0.1/127.0.0.1:41213:41213),(127.0.0.1/127.0.0.1:36425:36425)] 2024-12-03T06:21:59,511 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-03T06:21:59,512 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-03T06:21:59,515 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-03T06:21:59,520 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-03T06:21:59,524 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-03T06:21:59,525 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:21:59,525 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-03T06:21:59,525 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-03T06:21:59,529 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T06:21:59,530 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T06:21:59,531 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:21:59,531 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T06:21:59,532 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T06:21:59,533 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T06:21:59,534 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:21:59,535 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T06:21:59,535 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T06:21:59,536 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T06:21:59,536 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:21:59,537 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T06:21:59,537 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T06:21:59,539 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T06:21:59,539 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:21:59,540 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T06:21:59,540 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T06:21:59,541 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740 2024-12-03T06:21:59,545 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740 2024-12-03T06:21:59,547 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T06:21:59,547 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T06:21:59,548 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T06:21:59,551 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T06:21:59,552 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59233398, jitterRate=-0.11735358834266663}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T06:21:59,553 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-03T06:21:59,554 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733206919526Writing region info on filesystem at 1733206919526Initializing all the Stores at 1733206919528 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733206919528Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733206919529 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206919529Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733206919529Cleaning up temporary data from old regions at 1733206919547 (+18 ms)Running coprocessor post-open hooks at 1733206919553 (+6 ms)Region opened successfully at 1733206919554 (+1 ms) 2024-12-03T06:21:59,561 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733206919459 2024-12-03T06:21:59,572 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-03T06:21:59,573 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-03T06:21:59,575 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,37369,1733206917509 2024-12-03T06:21:59,577 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 122cddc95ec8,37369,1733206917509, state=OPEN 2024-12-03T06:21:59,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:21:59,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37369-0x1009eb0b9ca0003, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:21:59,585 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:21:59,586 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:21:59,586 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:21:59,586 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:21:59,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:21:59,586 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:21:59,586 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=122cddc95ec8,37369,1733206917509 2024-12-03T06:21:59,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-03T06:21:59,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,37369,1733206917509 in 299 msec 2024-12-03T06:21:59,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-03T06:21:59,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 813 msec 2024-12-03T06:21:59,603 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T06:21:59,603 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-03T06:21:59,623 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:21:59,624 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,37369,1733206917509, seqNum=-1] 2024-12-03T06:21:59,643 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:21:59,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43377, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:21:59,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1110 sec 2024-12-03T06:21:59,679 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733206919679, completionTime=-1 2024-12-03T06:21:59,683 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-03T06:21:59,683 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T06:21:59,708 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-03T06:21:59,708 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733206979708 2024-12-03T06:21:59,708 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733207039708 2024-12-03T06:21:59,708 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-12-03T06:21:59,710 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-03T06:21:59,717 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=122cddc95ec8,35025,1733206916647-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:59,717 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=122cddc95ec8,35025,1733206916647-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:59,717 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=122cddc95ec8,35025,1733206916647-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:59,719 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-122cddc95ec8:35025, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:59,719 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:59,720 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:59,726 DEBUG [master/122cddc95ec8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-03T06:21:59,756 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.125sec 2024-12-03T06:21:59,757 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-03T06:21:59,759 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-03T06:21:59,760 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-03T06:21:59,760 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-03T06:21:59,761 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-03T06:21:59,761 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=122cddc95ec8,35025,1733206916647-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T06:21:59,762 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=122cddc95ec8,35025,1733206916647-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-03T06:21:59,768 DEBUG [master/122cddc95ec8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-03T06:21:59,770 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-03T06:21:59,770 INFO [master/122cddc95ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=122cddc95ec8,35025,1733206916647-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T06:21:59,858 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39f0836e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:21:59,861 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-03T06:21:59,861 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-03T06:21:59,865 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 122cddc95ec8,35025,-1 for getting cluster id 2024-12-03T06:21:59,867 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T06:21:59,876 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c0c7c2db-4b33-4027-9625-bcb0de82bf43' 2024-12-03T06:21:59,879 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T06:21:59,879 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c0c7c2db-4b33-4027-9625-bcb0de82bf43" 2024-12-03T06:21:59,880 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37696b83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:21:59,880 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [122cddc95ec8,35025,-1] 2024-12-03T06:21:59,884 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T06:21:59,887 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:21:59,887 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T06:21:59,890 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41704ed8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T06:21:59,890 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:21:59,897 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,37369,1733206917509, seqNum=-1] 2024-12-03T06:21:59,898 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:21:59,900 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:21:59,942 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=122cddc95ec8,35025,1733206916647 2024-12-03T06:21:59,948 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T06:21:59,955 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 122cddc95ec8,35025,1733206916647 2024-12-03T06:21:59,958 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1c064dd6 2024-12-03T06:21:59,959 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T06:21:59,963 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T06:21:59,975 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T06:21:59,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-12-03T06:21:59,994 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T06:21:59,998 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-12-03T06:21:59,998 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:00,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T06:22:00,006 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T06:22:00,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741837_1013 (size=358) 2024-12-03T06:22:00,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741837_1013 (size=358) 2024-12-03T06:22:00,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741837_1013 (size=358) 2024-12-03T06:22:00,033 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 3bbc9d78d890c8d774a80646d9983ed6, NAME => 'test,ccc,1733206919969.3bbc9d78d890c8d774a80646d9983ed6.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:22:00,033 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 40b4b0967914401ebc70c24d4b56a381, NAME => 'test,bbb,1733206919969.40b4b0967914401ebc70c24d4b56a381.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:22:00,033 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 663ef4d3dbbcf7b1bb25bba1055da980, NAME => 'test,,1733206919969.663ef4d3dbbcf7b1bb25bba1055da980.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:22:00,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741839_1015 (size=45) 2024-12-03T06:22:00,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741839_1015 (size=45) 2024-12-03T06:22:00,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741839_1015 (size=45) 2024-12-03T06:22:00,061 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1733206919969.40b4b0967914401ebc70c24d4b56a381.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:00,061 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 40b4b0967914401ebc70c24d4b56a381, disabling compactions & flushes 2024-12-03T06:22:00,061 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1733206919969.40b4b0967914401ebc70c24d4b56a381. 2024-12-03T06:22:00,061 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1733206919969.40b4b0967914401ebc70c24d4b56a381. 2024-12-03T06:22:00,061 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1733206919969.40b4b0967914401ebc70c24d4b56a381. after waiting 0 ms 2024-12-03T06:22:00,061 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1733206919969.40b4b0967914401ebc70c24d4b56a381. 2024-12-03T06:22:00,061 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1733206919969.40b4b0967914401ebc70c24d4b56a381. 2024-12-03T06:22:00,061 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 40b4b0967914401ebc70c24d4b56a381: Waiting for close lock at 1733206920061Disabling compacts and flushes for region at 1733206920061Disabling writes for close at 1733206920061Writing region close event to WAL at 1733206920061Closed at 1733206920061 2024-12-03T06:22:00,062 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 33124cac09cf3a2502ce5e473bf614f4, NAME => 'test,ddd,1733206919969.33124cac09cf3a2502ce5e473bf614f4.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:22:00,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741838_1014 (size=45) 2024-12-03T06:22:00,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741838_1014 (size=45) 2024-12-03T06:22:00,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741838_1014 (size=45) 2024-12-03T06:22:00,071 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1733206919969.3bbc9d78d890c8d774a80646d9983ed6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:00,071 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 3bbc9d78d890c8d774a80646d9983ed6, disabling compactions & flushes 2024-12-03T06:22:00,071 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1733206919969.3bbc9d78d890c8d774a80646d9983ed6. 2024-12-03T06:22:00,071 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1733206919969.3bbc9d78d890c8d774a80646d9983ed6. 2024-12-03T06:22:00,071 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1733206919969.3bbc9d78d890c8d774a80646d9983ed6. after waiting 0 ms 2024-12-03T06:22:00,071 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1733206919969.3bbc9d78d890c8d774a80646d9983ed6. 2024-12-03T06:22:00,071 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1733206919969.3bbc9d78d890c8d774a80646d9983ed6. 2024-12-03T06:22:00,071 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 3bbc9d78d890c8d774a80646d9983ed6: Waiting for close lock at 1733206920071Disabling compacts and flushes for region at 1733206920071Disabling writes for close at 1733206920071Writing region close event to WAL at 1733206920071Closed at 1733206920071 2024-12-03T06:22:00,072 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => e884340ece2e3eb17a321ed6d089e5e7, NAME => 'test,eee,1733206919969.e884340ece2e3eb17a321ed6d089e5e7.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:22:00,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741840_1016 (size=42) 2024-12-03T06:22:00,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741840_1016 (size=42) 2024-12-03T06:22:00,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741840_1016 (size=42) 2024-12-03T06:22:00,087 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1733206919969.663ef4d3dbbcf7b1bb25bba1055da980.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:00,087 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 663ef4d3dbbcf7b1bb25bba1055da980, disabling compactions & flushes 2024-12-03T06:22:00,087 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1733206919969.663ef4d3dbbcf7b1bb25bba1055da980. 2024-12-03T06:22:00,087 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1733206919969.663ef4d3dbbcf7b1bb25bba1055da980. 2024-12-03T06:22:00,087 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1733206919969.663ef4d3dbbcf7b1bb25bba1055da980. after waiting 0 ms 2024-12-03T06:22:00,087 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1733206919969.663ef4d3dbbcf7b1bb25bba1055da980. 2024-12-03T06:22:00,087 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1733206919969.663ef4d3dbbcf7b1bb25bba1055da980. 2024-12-03T06:22:00,088 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 663ef4d3dbbcf7b1bb25bba1055da980: Waiting for close lock at 1733206920087Disabling compacts and flushes for region at 1733206920087Disabling writes for close at 1733206920087Writing region close event to WAL at 1733206920087Closed at 1733206920087 2024-12-03T06:22:00,088 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e48f9dd0d4bf7a7f5e5fb4dac950f301, NAME => 'test,fff,1733206919969.e48f9dd0d4bf7a7f5e5fb4dac950f301.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:22:00,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741841_1017 (size=45) 2024-12-03T06:22:00,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741841_1017 (size=45) 2024-12-03T06:22:00,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741841_1017 (size=45) 2024-12-03T06:22:00,099 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ddd,1733206919969.33124cac09cf3a2502ce5e473bf614f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:00,099 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 33124cac09cf3a2502ce5e473bf614f4, disabling compactions & flushes 2024-12-03T06:22:00,099 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ddd,1733206919969.33124cac09cf3a2502ce5e473bf614f4. 2024-12-03T06:22:00,099 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1733206919969.33124cac09cf3a2502ce5e473bf614f4. 2024-12-03T06:22:00,099 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1733206919969.33124cac09cf3a2502ce5e473bf614f4. after waiting 0 ms 2024-12-03T06:22:00,099 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1733206919969.33124cac09cf3a2502ce5e473bf614f4. 2024-12-03T06:22:00,099 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ddd,1733206919969.33124cac09cf3a2502ce5e473bf614f4. 2024-12-03T06:22:00,099 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 33124cac09cf3a2502ce5e473bf614f4: Waiting for close lock at 1733206920099Disabling compacts and flushes for region at 1733206920099Disabling writes for close at 1733206920099Writing region close event to WAL at 1733206920099Closed at 1733206920099 2024-12-03T06:22:00,100 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => ca916e60e459eeefe786a877b8700115, NAME => 'test,ggg,1733206919969.ca916e60e459eeefe786a877b8700115.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:22:00,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741842_1018 (size=45) 2024-12-03T06:22:00,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741842_1018 (size=45) 2024-12-03T06:22:00,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741842_1018 (size=45) 2024-12-03T06:22:00,112 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,eee,1733206919969.e884340ece2e3eb17a321ed6d089e5e7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:00,112 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing e884340ece2e3eb17a321ed6d089e5e7, disabling compactions & flushes 2024-12-03T06:22:00,112 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,eee,1733206919969.e884340ece2e3eb17a321ed6d089e5e7. 2024-12-03T06:22:00,112 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1733206919969.e884340ece2e3eb17a321ed6d089e5e7. 2024-12-03T06:22:00,112 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1733206919969.e884340ece2e3eb17a321ed6d089e5e7. after waiting 0 ms 2024-12-03T06:22:00,112 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1733206919969.e884340ece2e3eb17a321ed6d089e5e7. 2024-12-03T06:22:00,112 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,eee,1733206919969.e884340ece2e3eb17a321ed6d089e5e7. 2024-12-03T06:22:00,112 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for e884340ece2e3eb17a321ed6d089e5e7: Waiting for close lock at 1733206920112Disabling compacts and flushes for region at 1733206920112Disabling writes for close at 1733206920112Writing region close event to WAL at 1733206920112Closed at 1733206920112 2024-12-03T06:22:00,113 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 7d719355720f98b8bc93f39a6130075b, NAME => 'test,hhh,1733206919969.7d719355720f98b8bc93f39a6130075b.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:22:00,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T06:22:00,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741843_1019 (size=45) 2024-12-03T06:22:00,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741843_1019 (size=45) 2024-12-03T06:22:00,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741843_1019 (size=45) 2024-12-03T06:22:00,135 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,fff,1733206919969.e48f9dd0d4bf7a7f5e5fb4dac950f301.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:00,135 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing e48f9dd0d4bf7a7f5e5fb4dac950f301, disabling compactions & flushes 2024-12-03T06:22:00,135 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,fff,1733206919969.e48f9dd0d4bf7a7f5e5fb4dac950f301. 2024-12-03T06:22:00,135 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1733206919969.e48f9dd0d4bf7a7f5e5fb4dac950f301. 2024-12-03T06:22:00,135 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1733206919969.e48f9dd0d4bf7a7f5e5fb4dac950f301. after waiting 0 ms 2024-12-03T06:22:00,135 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1733206919969.e48f9dd0d4bf7a7f5e5fb4dac950f301. 2024-12-03T06:22:00,136 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,fff,1733206919969.e48f9dd0d4bf7a7f5e5fb4dac950f301. 2024-12-03T06:22:00,136 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for e48f9dd0d4bf7a7f5e5fb4dac950f301: Waiting for close lock at 1733206920135Disabling compacts and flushes for region at 1733206920135Disabling writes for close at 1733206920135Writing region close event to WAL at 1733206920136 (+1 ms)Closed at 1733206920136 2024-12-03T06:22:00,136 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a332f3da50b1f1fe35aa4e0307fe35f8, NAME => 'test,iii,1733206919969.a332f3da50b1f1fe35aa4e0307fe35f8.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:22:00,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741844_1020 (size=45) 2024-12-03T06:22:00,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741844_1020 (size=45) 2024-12-03T06:22:00,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741844_1020 (size=45) 2024-12-03T06:22:00,145 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ggg,1733206919969.ca916e60e459eeefe786a877b8700115.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:00,145 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing ca916e60e459eeefe786a877b8700115, disabling compactions & flushes 2024-12-03T06:22:00,145 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ggg,1733206919969.ca916e60e459eeefe786a877b8700115. 2024-12-03T06:22:00,145 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1733206919969.ca916e60e459eeefe786a877b8700115. 2024-12-03T06:22:00,145 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1733206919969.ca916e60e459eeefe786a877b8700115. after waiting 0 ms 2024-12-03T06:22:00,145 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1733206919969.ca916e60e459eeefe786a877b8700115. 2024-12-03T06:22:00,145 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ggg,1733206919969.ca916e60e459eeefe786a877b8700115. 2024-12-03T06:22:00,145 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for ca916e60e459eeefe786a877b8700115: Waiting for close lock at 1733206920145Disabling compacts and flushes for region at 1733206920145Disabling writes for close at 1733206920145Writing region close event to WAL at 1733206920145Closed at 1733206920145 2024-12-03T06:22:00,146 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 3c9e860d892b47160f56d24cb24e6010, NAME => 'test,jjj,1733206919969.3c9e860d892b47160f56d24cb24e6010.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:22:00,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741845_1021 (size=45) 2024-12-03T06:22:00,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741845_1021 (size=45) 2024-12-03T06:22:00,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741845_1021 (size=45) 2024-12-03T06:22:00,165 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,hhh,1733206919969.7d719355720f98b8bc93f39a6130075b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:00,165 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 7d719355720f98b8bc93f39a6130075b, disabling compactions & flushes 2024-12-03T06:22:00,165 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,hhh,1733206919969.7d719355720f98b8bc93f39a6130075b. 2024-12-03T06:22:00,165 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1733206919969.7d719355720f98b8bc93f39a6130075b. 2024-12-03T06:22:00,165 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1733206919969.7d719355720f98b8bc93f39a6130075b. after waiting 0 ms 2024-12-03T06:22:00,165 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1733206919969.7d719355720f98b8bc93f39a6130075b. 2024-12-03T06:22:00,165 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,hhh,1733206919969.7d719355720f98b8bc93f39a6130075b. 2024-12-03T06:22:00,165 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 7d719355720f98b8bc93f39a6130075b: Waiting for close lock at 1733206920165Disabling compacts and flushes for region at 1733206920165Disabling writes for close at 1733206920165Writing region close event to WAL at 1733206920165Closed at 1733206920165 2024-12-03T06:22:00,166 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => f495460f3e799a3e64378f8879c721bb, NAME => 'test,kkk,1733206919969.f495460f3e799a3e64378f8879c721bb.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:22:00,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741846_1022 (size=45) 2024-12-03T06:22:00,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741846_1022 (size=45) 2024-12-03T06:22:00,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741846_1022 (size=45) 2024-12-03T06:22:00,181 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,iii,1733206919969.a332f3da50b1f1fe35aa4e0307fe35f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:00,181 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing a332f3da50b1f1fe35aa4e0307fe35f8, disabling compactions & flushes 2024-12-03T06:22:00,181 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,iii,1733206919969.a332f3da50b1f1fe35aa4e0307fe35f8. 2024-12-03T06:22:00,181 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1733206919969.a332f3da50b1f1fe35aa4e0307fe35f8. 2024-12-03T06:22:00,181 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1733206919969.a332f3da50b1f1fe35aa4e0307fe35f8. after waiting 0 ms 2024-12-03T06:22:00,181 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1733206919969.a332f3da50b1f1fe35aa4e0307fe35f8. 2024-12-03T06:22:00,181 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,iii,1733206919969.a332f3da50b1f1fe35aa4e0307fe35f8. 2024-12-03T06:22:00,181 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for a332f3da50b1f1fe35aa4e0307fe35f8: Waiting for close lock at 1733206920181Disabling compacts and flushes for region at 1733206920181Disabling writes for close at 1733206920181Writing region close event to WAL at 1733206920181Closed at 1733206920181 2024-12-03T06:22:00,182 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1ddf2a494d8d0fcd31eac685512f5644, NAME => 'test,lll,1733206919969.1ddf2a494d8d0fcd31eac685512f5644.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:22:00,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741847_1023 (size=45) 2024-12-03T06:22:00,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741847_1023 (size=45) 2024-12-03T06:22:00,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741847_1023 (size=45) 2024-12-03T06:22:00,188 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,jjj,1733206919969.3c9e860d892b47160f56d24cb24e6010.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:00,189 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 3c9e860d892b47160f56d24cb24e6010, disabling compactions & flushes 2024-12-03T06:22:00,189 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,jjj,1733206919969.3c9e860d892b47160f56d24cb24e6010. 2024-12-03T06:22:00,189 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1733206919969.3c9e860d892b47160f56d24cb24e6010. 2024-12-03T06:22:00,189 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1733206919969.3c9e860d892b47160f56d24cb24e6010. after waiting 0 ms 2024-12-03T06:22:00,189 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1733206919969.3c9e860d892b47160f56d24cb24e6010. 2024-12-03T06:22:00,189 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,jjj,1733206919969.3c9e860d892b47160f56d24cb24e6010. 2024-12-03T06:22:00,189 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 3c9e860d892b47160f56d24cb24e6010: Waiting for close lock at 1733206920188Disabling compacts and flushes for region at 1733206920188Disabling writes for close at 1733206920189 (+1 ms)Writing region close event to WAL at 1733206920189Closed at 1733206920189 2024-12-03T06:22:00,190 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 050ab97be4cc9f90a155d3b261335a0d, NAME => 'test,mmm,1733206919969.050ab97be4cc9f90a155d3b261335a0d.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:22:00,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741848_1024 (size=45) 2024-12-03T06:22:00,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741848_1024 (size=45) 2024-12-03T06:22:00,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741848_1024 (size=45) 2024-12-03T06:22:00,207 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,kkk,1733206919969.f495460f3e799a3e64378f8879c721bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:00,207 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing f495460f3e799a3e64378f8879c721bb, disabling compactions & flushes 2024-12-03T06:22:00,207 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,kkk,1733206919969.f495460f3e799a3e64378f8879c721bb. 2024-12-03T06:22:00,207 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1733206919969.f495460f3e799a3e64378f8879c721bb. 2024-12-03T06:22:00,207 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1733206919969.f495460f3e799a3e64378f8879c721bb. after waiting 0 ms 2024-12-03T06:22:00,207 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1733206919969.f495460f3e799a3e64378f8879c721bb. 2024-12-03T06:22:00,207 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,kkk,1733206919969.f495460f3e799a3e64378f8879c721bb. 2024-12-03T06:22:00,207 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for f495460f3e799a3e64378f8879c721bb: Waiting for close lock at 1733206920207Disabling compacts and flushes for region at 1733206920207Disabling writes for close at 1733206920207Writing region close event to WAL at 1733206920207Closed at 1733206920207 2024-12-03T06:22:00,208 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => b5f7ab205e6c897503e3ec729aaa3170, NAME => 'test,nnn,1733206919969.b5f7ab205e6c897503e3ec729aaa3170.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:22:00,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741849_1025 (size=45) 2024-12-03T06:22:00,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741849_1025 (size=45) 2024-12-03T06:22:00,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741849_1025 (size=45) 2024-12-03T06:22:00,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741850_1026 (size=45) 2024-12-03T06:22:00,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741850_1026 (size=45) 2024-12-03T06:22:00,221 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,lll,1733206919969.1ddf2a494d8d0fcd31eac685512f5644.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:00,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741850_1026 (size=45) 2024-12-03T06:22:00,221 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 1ddf2a494d8d0fcd31eac685512f5644, disabling compactions & flushes 2024-12-03T06:22:00,221 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,lll,1733206919969.1ddf2a494d8d0fcd31eac685512f5644. 2024-12-03T06:22:00,221 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1733206919969.1ddf2a494d8d0fcd31eac685512f5644. 2024-12-03T06:22:00,221 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1733206919969.1ddf2a494d8d0fcd31eac685512f5644. after waiting 0 ms 2024-12-03T06:22:00,221 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1733206919969.1ddf2a494d8d0fcd31eac685512f5644. 2024-12-03T06:22:00,221 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,lll,1733206919969.1ddf2a494d8d0fcd31eac685512f5644. 2024-12-03T06:22:00,221 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1ddf2a494d8d0fcd31eac685512f5644: Waiting for close lock at 1733206920221Disabling compacts and flushes for region at 1733206920221Disabling writes for close at 1733206920221Writing region close event to WAL at 1733206920221Closed at 1733206920221 2024-12-03T06:22:00,222 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a6fcc2e40df410122347a4873dfd1129, NAME => 'test,ooo,1733206919969.a6fcc2e40df410122347a4873dfd1129.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:22:00,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741851_1027 (size=45) 2024-12-03T06:22:00,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741851_1027 (size=45) 2024-12-03T06:22:00,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741851_1027 (size=45) 2024-12-03T06:22:00,228 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,nnn,1733206919969.b5f7ab205e6c897503e3ec729aaa3170.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:00,228 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing b5f7ab205e6c897503e3ec729aaa3170, disabling compactions & flushes 2024-12-03T06:22:00,228 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,nnn,1733206919969.b5f7ab205e6c897503e3ec729aaa3170. 2024-12-03T06:22:00,229 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1733206919969.b5f7ab205e6c897503e3ec729aaa3170. 2024-12-03T06:22:00,229 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1733206919969.b5f7ab205e6c897503e3ec729aaa3170. after waiting 0 ms 2024-12-03T06:22:00,229 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1733206919969.b5f7ab205e6c897503e3ec729aaa3170. 2024-12-03T06:22:00,229 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,nnn,1733206919969.b5f7ab205e6c897503e3ec729aaa3170. 2024-12-03T06:22:00,229 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for b5f7ab205e6c897503e3ec729aaa3170: Waiting for close lock at 1733206920228Disabling compacts and flushes for region at 1733206920228Disabling writes for close at 1733206920229 (+1 ms)Writing region close event to WAL at 1733206920229Closed at 1733206920229 2024-12-03T06:22:00,229 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 07dbe58fce62a9e98b77016cec645f5e, NAME => 'test,ppp,1733206919969.07dbe58fce62a9e98b77016cec645f5e.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:22:00,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741852_1028 (size=45) 2024-12-03T06:22:00,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741852_1028 (size=45) 2024-12-03T06:22:00,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741852_1028 (size=45) 2024-12-03T06:22:00,238 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ooo,1733206919969.a6fcc2e40df410122347a4873dfd1129.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:00,238 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing a6fcc2e40df410122347a4873dfd1129, disabling compactions & flushes 2024-12-03T06:22:00,238 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ooo,1733206919969.a6fcc2e40df410122347a4873dfd1129. 2024-12-03T06:22:00,238 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1733206919969.a6fcc2e40df410122347a4873dfd1129. 2024-12-03T06:22:00,238 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1733206919969.a6fcc2e40df410122347a4873dfd1129. after waiting 0 ms 2024-12-03T06:22:00,238 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1733206919969.a6fcc2e40df410122347a4873dfd1129. 2024-12-03T06:22:00,238 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ooo,1733206919969.a6fcc2e40df410122347a4873dfd1129. 2024-12-03T06:22:00,238 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for a6fcc2e40df410122347a4873dfd1129: Waiting for close lock at 1733206920238Disabling compacts and flushes for region at 1733206920238Disabling writes for close at 1733206920238Writing region close event to WAL at 1733206920238Closed at 1733206920238 2024-12-03T06:22:00,239 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6badf4f9ef3213a92a5f6ace694c6482, NAME => 'test,qqq,1733206919969.6badf4f9ef3213a92a5f6ace694c6482.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:22:00,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741853_1029 (size=45) 2024-12-03T06:22:00,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741853_1029 (size=45) 2024-12-03T06:22:00,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741853_1029 (size=45) 2024-12-03T06:22:00,244 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ppp,1733206919969.07dbe58fce62a9e98b77016cec645f5e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:00,244 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 07dbe58fce62a9e98b77016cec645f5e, disabling compactions & flushes 2024-12-03T06:22:00,244 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ppp,1733206919969.07dbe58fce62a9e98b77016cec645f5e. 2024-12-03T06:22:00,244 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1733206919969.07dbe58fce62a9e98b77016cec645f5e. 2024-12-03T06:22:00,244 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1733206919969.07dbe58fce62a9e98b77016cec645f5e. after waiting 0 ms 2024-12-03T06:22:00,244 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1733206919969.07dbe58fce62a9e98b77016cec645f5e. 2024-12-03T06:22:00,245 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ppp,1733206919969.07dbe58fce62a9e98b77016cec645f5e. 2024-12-03T06:22:00,245 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 07dbe58fce62a9e98b77016cec645f5e: Waiting for close lock at 1733206920244Disabling compacts and flushes for region at 1733206920244Disabling writes for close at 1733206920244Writing region close event to WAL at 1733206920244Closed at 1733206920245 (+1 ms) 2024-12-03T06:22:00,245 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => afa4ab371a599cc21eb81082233626e3, NAME => 'test,rrr,1733206919969.afa4ab371a599cc21eb81082233626e3.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:22:00,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741854_1030 (size=45) 2024-12-03T06:22:00,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741854_1030 (size=45) 2024-12-03T06:22:00,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741854_1030 (size=45) 2024-12-03T06:22:00,253 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,qqq,1733206919969.6badf4f9ef3213a92a5f6ace694c6482.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:00,253 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 6badf4f9ef3213a92a5f6ace694c6482, disabling compactions & flushes 2024-12-03T06:22:00,253 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,qqq,1733206919969.6badf4f9ef3213a92a5f6ace694c6482. 2024-12-03T06:22:00,254 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1733206919969.6badf4f9ef3213a92a5f6ace694c6482. 2024-12-03T06:22:00,254 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1733206919969.6badf4f9ef3213a92a5f6ace694c6482. after waiting 0 ms 2024-12-03T06:22:00,254 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1733206919969.6badf4f9ef3213a92a5f6ace694c6482. 2024-12-03T06:22:00,254 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,qqq,1733206919969.6badf4f9ef3213a92a5f6ace694c6482. 2024-12-03T06:22:00,254 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6badf4f9ef3213a92a5f6ace694c6482: Waiting for close lock at 1733206920253Disabling compacts and flushes for region at 1733206920253Disabling writes for close at 1733206920254 (+1 ms)Writing region close event to WAL at 1733206920254Closed at 1733206920254 2024-12-03T06:22:00,255 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4e8636a0790f504025df15b4adf65735, NAME => 'test,sss,1733206919969.4e8636a0790f504025df15b4adf65735.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:22:00,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741855_1031 (size=45) 2024-12-03T06:22:00,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741855_1031 (size=45) 2024-12-03T06:22:00,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741855_1031 (size=45) 2024-12-03T06:22:00,259 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,rrr,1733206919969.afa4ab371a599cc21eb81082233626e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:00,259 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing afa4ab371a599cc21eb81082233626e3, disabling compactions & flushes 2024-12-03T06:22:00,259 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,rrr,1733206919969.afa4ab371a599cc21eb81082233626e3. 2024-12-03T06:22:00,259 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1733206919969.afa4ab371a599cc21eb81082233626e3. 2024-12-03T06:22:00,259 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1733206919969.afa4ab371a599cc21eb81082233626e3. after waiting 0 ms 2024-12-03T06:22:00,259 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1733206919969.afa4ab371a599cc21eb81082233626e3. 2024-12-03T06:22:00,259 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,rrr,1733206919969.afa4ab371a599cc21eb81082233626e3. 2024-12-03T06:22:00,259 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for afa4ab371a599cc21eb81082233626e3: Waiting for close lock at 1733206920259Disabling compacts and flushes for region at 1733206920259Disabling writes for close at 1733206920259Writing region close event to WAL at 1733206920259Closed at 1733206920259 2024-12-03T06:22:00,260 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 1e061778c0efa365f63836d0691ff7a8, NAME => 'test,ttt,1733206919969.1e061778c0efa365f63836d0691ff7a8.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:22:00,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741856_1032 (size=45) 2024-12-03T06:22:00,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741856_1032 (size=45) 2024-12-03T06:22:00,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741856_1032 (size=45) 2024-12-03T06:22:00,270 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,sss,1733206919969.4e8636a0790f504025df15b4adf65735.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:00,271 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 4e8636a0790f504025df15b4adf65735, disabling compactions & flushes 2024-12-03T06:22:00,271 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,sss,1733206919969.4e8636a0790f504025df15b4adf65735. 2024-12-03T06:22:00,271 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1733206919969.4e8636a0790f504025df15b4adf65735. 2024-12-03T06:22:00,271 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1733206919969.4e8636a0790f504025df15b4adf65735. after waiting 0 ms 2024-12-03T06:22:00,271 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1733206919969.4e8636a0790f504025df15b4adf65735. 2024-12-03T06:22:00,271 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,sss,1733206919969.4e8636a0790f504025df15b4adf65735. 2024-12-03T06:22:00,271 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4e8636a0790f504025df15b4adf65735: Waiting for close lock at 1733206920271Disabling compacts and flushes for region at 1733206920271Disabling writes for close at 1733206920271Writing region close event to WAL at 1733206920271Closed at 1733206920271 2024-12-03T06:22:00,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741857_1033 (size=45) 2024-12-03T06:22:00,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741857_1033 (size=45) 2024-12-03T06:22:00,272 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e3e1e4a86d09a1acb1fcf9d92f129898, NAME => 'test,uuu,1733206919969.e3e1e4a86d09a1acb1fcf9d92f129898.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:22:00,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741857_1033 (size=45) 2024-12-03T06:22:00,274 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ttt,1733206919969.1e061778c0efa365f63836d0691ff7a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:00,274 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 1e061778c0efa365f63836d0691ff7a8, disabling compactions & flushes 2024-12-03T06:22:00,274 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ttt,1733206919969.1e061778c0efa365f63836d0691ff7a8. 2024-12-03T06:22:00,274 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1733206919969.1e061778c0efa365f63836d0691ff7a8. 2024-12-03T06:22:00,274 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1733206919969.1e061778c0efa365f63836d0691ff7a8. after waiting 0 ms 2024-12-03T06:22:00,274 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1733206919969.1e061778c0efa365f63836d0691ff7a8. 2024-12-03T06:22:00,274 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ttt,1733206919969.1e061778c0efa365f63836d0691ff7a8. 2024-12-03T06:22:00,274 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 1e061778c0efa365f63836d0691ff7a8: Waiting for close lock at 1733206920274Disabling compacts and flushes for region at 1733206920274Disabling writes for close at 1733206920274Writing region close event to WAL at 1733206920274Closed at 1733206920274 2024-12-03T06:22:00,275 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 923ece5c25680df02ff5f922d838bc62, NAME => 'test,vvv,1733206919969.923ece5c25680df02ff5f922d838bc62.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:22:00,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741858_1034 (size=45) 2024-12-03T06:22:00,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741858_1034 (size=45) 2024-12-03T06:22:00,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741858_1034 (size=45) 2024-12-03T06:22:00,287 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,uuu,1733206919969.e3e1e4a86d09a1acb1fcf9d92f129898.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:00,287 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing e3e1e4a86d09a1acb1fcf9d92f129898, disabling compactions & flushes 2024-12-03T06:22:00,288 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,uuu,1733206919969.e3e1e4a86d09a1acb1fcf9d92f129898. 2024-12-03T06:22:00,288 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1733206919969.e3e1e4a86d09a1acb1fcf9d92f129898. 2024-12-03T06:22:00,288 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1733206919969.e3e1e4a86d09a1acb1fcf9d92f129898. after waiting 0 ms 2024-12-03T06:22:00,288 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1733206919969.e3e1e4a86d09a1acb1fcf9d92f129898. 2024-12-03T06:22:00,288 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,uuu,1733206919969.e3e1e4a86d09a1acb1fcf9d92f129898. 2024-12-03T06:22:00,288 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for e3e1e4a86d09a1acb1fcf9d92f129898: Waiting for close lock at 1733206920287Disabling compacts and flushes for region at 1733206920287Disabling writes for close at 1733206920288 (+1 ms)Writing region close event to WAL at 1733206920288Closed at 1733206920288 2024-12-03T06:22:00,289 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e9f6e863b0ed7a188cb723f7a298ba2c, NAME => 'test,www,1733206919969.e9f6e863b0ed7a188cb723f7a298ba2c.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:22:00,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741859_1035 (size=45) 2024-12-03T06:22:00,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741859_1035 (size=45) 2024-12-03T06:22:00,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741859_1035 (size=45) 2024-12-03T06:22:00,297 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,vvv,1733206919969.923ece5c25680df02ff5f922d838bc62.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:00,297 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 923ece5c25680df02ff5f922d838bc62, disabling compactions & flushes 2024-12-03T06:22:00,297 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,vvv,1733206919969.923ece5c25680df02ff5f922d838bc62. 2024-12-03T06:22:00,297 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1733206919969.923ece5c25680df02ff5f922d838bc62. 2024-12-03T06:22:00,297 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1733206919969.923ece5c25680df02ff5f922d838bc62. after waiting 0 ms 2024-12-03T06:22:00,297 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1733206919969.923ece5c25680df02ff5f922d838bc62. 2024-12-03T06:22:00,297 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,vvv,1733206919969.923ece5c25680df02ff5f922d838bc62. 2024-12-03T06:22:00,297 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 923ece5c25680df02ff5f922d838bc62: Waiting for close lock at 1733206920297Disabling compacts and flushes for region at 1733206920297Disabling writes for close at 1733206920297Writing region close event to WAL at 1733206920297Closed at 1733206920297 2024-12-03T06:22:00,298 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 6b148e6fa0896eea84e6c38d5293e44a, NAME => 'test,xxx,1733206919969.6b148e6fa0896eea84e6c38d5293e44a.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:22:00,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741860_1036 (size=45) 2024-12-03T06:22:00,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741860_1036 (size=45) 2024-12-03T06:22:00,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741860_1036 (size=45) 2024-12-03T06:22:00,306 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,www,1733206919969.e9f6e863b0ed7a188cb723f7a298ba2c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:00,306 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing e9f6e863b0ed7a188cb723f7a298ba2c, disabling compactions & flushes 2024-12-03T06:22:00,306 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,www,1733206919969.e9f6e863b0ed7a188cb723f7a298ba2c. 2024-12-03T06:22:00,306 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1733206919969.e9f6e863b0ed7a188cb723f7a298ba2c. 2024-12-03T06:22:00,306 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1733206919969.e9f6e863b0ed7a188cb723f7a298ba2c. after waiting 0 ms 2024-12-03T06:22:00,306 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1733206919969.e9f6e863b0ed7a188cb723f7a298ba2c. 2024-12-03T06:22:00,306 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,www,1733206919969.e9f6e863b0ed7a188cb723f7a298ba2c. 2024-12-03T06:22:00,306 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for e9f6e863b0ed7a188cb723f7a298ba2c: Waiting for close lock at 1733206920306Disabling compacts and flushes for region at 1733206920306Disabling writes for close at 1733206920306Writing region close event to WAL at 1733206920306Closed at 1733206920306 2024-12-03T06:22:00,307 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ae2330e378b6ad4b1a3e345529a0b27f, NAME => 'test,yyy,1733206919969.ae2330e378b6ad4b1a3e345529a0b27f.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:22:00,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741861_1037 (size=45) 2024-12-03T06:22:00,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741861_1037 (size=45) 2024-12-03T06:22:00,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741861_1037 (size=45) 2024-12-03T06:22:00,312 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,xxx,1733206919969.6b148e6fa0896eea84e6c38d5293e44a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:00,312 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 6b148e6fa0896eea84e6c38d5293e44a, disabling compactions & flushes 2024-12-03T06:22:00,312 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,xxx,1733206919969.6b148e6fa0896eea84e6c38d5293e44a. 2024-12-03T06:22:00,312 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1733206919969.6b148e6fa0896eea84e6c38d5293e44a. 2024-12-03T06:22:00,312 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1733206919969.6b148e6fa0896eea84e6c38d5293e44a. after waiting 0 ms 2024-12-03T06:22:00,312 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1733206919969.6b148e6fa0896eea84e6c38d5293e44a. 2024-12-03T06:22:00,312 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,xxx,1733206919969.6b148e6fa0896eea84e6c38d5293e44a. 2024-12-03T06:22:00,312 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 6b148e6fa0896eea84e6c38d5293e44a: Waiting for close lock at 1733206920312Disabling compacts and flushes for region at 1733206920312Disabling writes for close at 1733206920312Writing region close event to WAL at 1733206920312Closed at 1733206920312 2024-12-03T06:22:00,313 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 671297fbd74543fd2b2deed06d08e6a7, NAME => 'test,zzz,1733206919969.671297fbd74543fd2b2deed06d08e6a7.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261 2024-12-03T06:22:00,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741862_1038 (size=45) 2024-12-03T06:22:00,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741862_1038 (size=45) 2024-12-03T06:22:00,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741862_1038 (size=45) 2024-12-03T06:22:00,319 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,yyy,1733206919969.ae2330e378b6ad4b1a3e345529a0b27f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:00,319 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing ae2330e378b6ad4b1a3e345529a0b27f, disabling compactions & flushes 2024-12-03T06:22:00,319 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,yyy,1733206919969.ae2330e378b6ad4b1a3e345529a0b27f. 2024-12-03T06:22:00,319 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1733206919969.ae2330e378b6ad4b1a3e345529a0b27f. 2024-12-03T06:22:00,319 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1733206919969.ae2330e378b6ad4b1a3e345529a0b27f. after waiting 0 ms 2024-12-03T06:22:00,319 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1733206919969.ae2330e378b6ad4b1a3e345529a0b27f. 2024-12-03T06:22:00,319 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,yyy,1733206919969.ae2330e378b6ad4b1a3e345529a0b27f. 2024-12-03T06:22:00,319 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for ae2330e378b6ad4b1a3e345529a0b27f: Waiting for close lock at 1733206920319Disabling compacts and flushes for region at 1733206920319Disabling writes for close at 1733206920319Writing region close event to WAL at 1733206920319Closed at 1733206920319 2024-12-03T06:22:00,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741863_1039 (size=42) 2024-12-03T06:22:00,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741863_1039 (size=42) 2024-12-03T06:22:00,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741863_1039 (size=42) 2024-12-03T06:22:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T06:22:00,325 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,zzz,1733206919969.671297fbd74543fd2b2deed06d08e6a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:00,325 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 671297fbd74543fd2b2deed06d08e6a7, disabling compactions & flushes 2024-12-03T06:22:00,325 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,zzz,1733206919969.671297fbd74543fd2b2deed06d08e6a7. 2024-12-03T06:22:00,325 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1733206919969.671297fbd74543fd2b2deed06d08e6a7. 2024-12-03T06:22:00,325 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1733206919969.671297fbd74543fd2b2deed06d08e6a7. after waiting 0 ms 2024-12-03T06:22:00,325 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1733206919969.671297fbd74543fd2b2deed06d08e6a7. 2024-12-03T06:22:00,325 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,zzz,1733206919969.671297fbd74543fd2b2deed06d08e6a7. 2024-12-03T06:22:00,325 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 671297fbd74543fd2b2deed06d08e6a7: Waiting for close lock at 1733206920325Disabling compacts and flushes for region at 1733206920325Disabling writes for close at 1733206920325Writing region close event to WAL at 1733206920325Closed at 1733206920325 2024-12-03T06:22:00,624 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,mmm,1733206919969.050ab97be4cc9f90a155d3b261335a0d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:00,624 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 050ab97be4cc9f90a155d3b261335a0d, disabling compactions & flushes 2024-12-03T06:22:00,624 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,mmm,1733206919969.050ab97be4cc9f90a155d3b261335a0d. 2024-12-03T06:22:00,625 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1733206919969.050ab97be4cc9f90a155d3b261335a0d. 2024-12-03T06:22:00,625 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1733206919969.050ab97be4cc9f90a155d3b261335a0d. after waiting 0 ms 2024-12-03T06:22:00,625 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1733206919969.050ab97be4cc9f90a155d3b261335a0d. 2024-12-03T06:22:00,625 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,mmm,1733206919969.050ab97be4cc9f90a155d3b261335a0d. 2024-12-03T06:22:00,625 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 050ab97be4cc9f90a155d3b261335a0d: Waiting for close lock at 1733206920624Disabling compacts and flushes for region at 1733206920624Disabling writes for close at 1733206920625 (+1 ms)Writing region close event to WAL at 1733206920625Closed at 1733206920625 2024-12-03T06:22:00,628 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T06:22:00,633 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1733206919969.40b4b0967914401ebc70c24d4b56a381.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733206920629"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206920629"}]},"ts":"1733206920629"} 2024-12-03T06:22:00,634 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1733206919969.3bbc9d78d890c8d774a80646d9983ed6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733206920629"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206920629"}]},"ts":"1733206920629"} 2024-12-03T06:22:00,634 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1733206919969.663ef4d3dbbcf7b1bb25bba1055da980.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733206920629"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206920629"}]},"ts":"1733206920629"} 2024-12-03T06:22:00,634 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1733206919969.33124cac09cf3a2502ce5e473bf614f4.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733206920629"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206920629"}]},"ts":"1733206920629"} 2024-12-03T06:22:00,634 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1733206919969.e884340ece2e3eb17a321ed6d089e5e7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733206920629"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206920629"}]},"ts":"1733206920629"} 2024-12-03T06:22:00,634 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1733206919969.e48f9dd0d4bf7a7f5e5fb4dac950f301.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733206920629"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206920629"}]},"ts":"1733206920629"} 2024-12-03T06:22:00,634 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1733206919969.ca916e60e459eeefe786a877b8700115.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733206920629"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206920629"}]},"ts":"1733206920629"} 2024-12-03T06:22:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T06:22:00,634 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1733206919969.7d719355720f98b8bc93f39a6130075b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733206920629"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206920629"}]},"ts":"1733206920629"} 2024-12-03T06:22:00,634 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1733206919969.a332f3da50b1f1fe35aa4e0307fe35f8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733206920629"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206920629"}]},"ts":"1733206920629"} 2024-12-03T06:22:00,635 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1733206919969.3c9e860d892b47160f56d24cb24e6010.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733206920629"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206920629"}]},"ts":"1733206920629"} 2024-12-03T06:22:00,635 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1733206919969.f495460f3e799a3e64378f8879c721bb.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733206920629"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206920629"}]},"ts":"1733206920629"} 2024-12-03T06:22:00,635 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1733206919969.1ddf2a494d8d0fcd31eac685512f5644.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733206920629"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206920629"}]},"ts":"1733206920629"} 2024-12-03T06:22:00,635 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1733206919969.b5f7ab205e6c897503e3ec729aaa3170.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733206920629"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206920629"}]},"ts":"1733206920629"} 2024-12-03T06:22:00,635 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1733206919969.a6fcc2e40df410122347a4873dfd1129.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733206920629"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206920629"}]},"ts":"1733206920629"} 2024-12-03T06:22:00,635 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1733206919969.07dbe58fce62a9e98b77016cec645f5e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733206920629"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206920629"}]},"ts":"1733206920629"} 2024-12-03T06:22:00,635 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1733206919969.6badf4f9ef3213a92a5f6ace694c6482.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733206920629"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206920629"}]},"ts":"1733206920629"} 2024-12-03T06:22:00,635 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1733206919969.afa4ab371a599cc21eb81082233626e3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733206920629"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206920629"}]},"ts":"1733206920629"} 2024-12-03T06:22:00,635 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1733206919969.4e8636a0790f504025df15b4adf65735.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733206920629"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206920629"}]},"ts":"1733206920629"} 2024-12-03T06:22:00,635 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1733206919969.1e061778c0efa365f63836d0691ff7a8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733206920629"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206920629"}]},"ts":"1733206920629"} 2024-12-03T06:22:00,636 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1733206919969.e3e1e4a86d09a1acb1fcf9d92f129898.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733206920629"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206920629"}]},"ts":"1733206920629"} 2024-12-03T06:22:00,636 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1733206919969.923ece5c25680df02ff5f922d838bc62.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733206920629"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206920629"}]},"ts":"1733206920629"} 2024-12-03T06:22:00,636 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1733206919969.e9f6e863b0ed7a188cb723f7a298ba2c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733206920629"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206920629"}]},"ts":"1733206920629"} 2024-12-03T06:22:00,636 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1733206919969.6b148e6fa0896eea84e6c38d5293e44a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733206920629"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206920629"}]},"ts":"1733206920629"} 2024-12-03T06:22:00,636 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1733206919969.ae2330e378b6ad4b1a3e345529a0b27f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733206920629"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206920629"}]},"ts":"1733206920629"} 2024-12-03T06:22:00,636 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1733206919969.671297fbd74543fd2b2deed06d08e6a7.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733206920629"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206920629"}]},"ts":"1733206920629"} 2024-12-03T06:22:00,636 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1733206919969.050ab97be4cc9f90a155d3b261335a0d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733206920629"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733206920629"}]},"ts":"1733206920629"} 2024-12-03T06:22:00,674 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-12-03T06:22:00,677 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T06:22:00,679 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733206920677"}]},"ts":"1733206920677"} 2024-12-03T06:22:00,684 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-12-03T06:22:00,685 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {122cddc95ec8=0} racks are {/default-rack=0} 2024-12-03T06:22:00,703 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T06:22:00,703 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T06:22:00,703 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-03T06:22:00,703 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T06:22:00,703 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T06:22:00,703 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-03T06:22:00,704 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T06:22:00,704 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T06:22:00,704 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-03T06:22:00,704 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T06:22:00,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=40b4b0967914401ebc70c24d4b56a381, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3bbc9d78d890c8d774a80646d9983ed6, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=33124cac09cf3a2502ce5e473bf614f4, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e884340ece2e3eb17a321ed6d089e5e7, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ca916e60e459eeefe786a877b8700115, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7d719355720f98b8bc93f39a6130075b, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3c9e860d892b47160f56d24cb24e6010, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f495460f3e799a3e64378f8879c721bb, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=050ab97be4cc9f90a155d3b261335a0d, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b5f7ab205e6c897503e3ec729aaa3170, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a6fcc2e40df410122347a4873dfd1129, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6badf4f9ef3213a92a5f6ace694c6482, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=afa4ab371a599cc21eb81082233626e3, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1e061778c0efa365f63836d0691ff7a8, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e3e1e4a86d09a1acb1fcf9d92f129898, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e9f6e863b0ed7a188cb723f7a298ba2c, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6b148e6fa0896eea84e6c38d5293e44a, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=671297fbd74543fd2b2deed06d08e6a7, ASSIGN}] 2024-12-03T06:22:00,714 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=671297fbd74543fd2b2deed06d08e6a7, ASSIGN 2024-12-03T06:22:00,716 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=671297fbd74543fd2b2deed06d08e6a7, ASSIGN; state=OFFLINE, location=122cddc95ec8,43629,1733206917367; forceNewPlan=false, retain=false 2024-12-03T06:22:00,717 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:00,718 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN; state=OFFLINE, location=122cddc95ec8,37369,1733206917509; forceNewPlan=false, retain=false 2024-12-03T06:22:00,719 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6b148e6fa0896eea84e6c38d5293e44a, ASSIGN 2024-12-03T06:22:00,721 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6b148e6fa0896eea84e6c38d5293e44a, ASSIGN; state=OFFLINE, location=122cddc95ec8,45577,1733206917466; forceNewPlan=false, retain=false 2024-12-03T06:22:00,722 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e9f6e863b0ed7a188cb723f7a298ba2c, ASSIGN 2024-12-03T06:22:00,724 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e9f6e863b0ed7a188cb723f7a298ba2c, ASSIGN; state=OFFLINE, location=122cddc95ec8,43629,1733206917367; forceNewPlan=false, retain=false 2024-12-03T06:22:00,726 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN 2024-12-03T06:22:00,728 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN; state=OFFLINE, location=122cddc95ec8,37369,1733206917509; forceNewPlan=false, retain=false 2024-12-03T06:22:00,730 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e3e1e4a86d09a1acb1fcf9d92f129898, ASSIGN 2024-12-03T06:22:00,732 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e3e1e4a86d09a1acb1fcf9d92f129898, ASSIGN; state=OFFLINE, location=122cddc95ec8,45577,1733206917466; forceNewPlan=false, retain=false 2024-12-03T06:22:00,733 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1e061778c0efa365f63836d0691ff7a8, ASSIGN 2024-12-03T06:22:00,735 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1e061778c0efa365f63836d0691ff7a8, ASSIGN; state=OFFLINE, location=122cddc95ec8,43629,1733206917367; forceNewPlan=false, retain=false 2024-12-03T06:22:00,736 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:00,738 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN; state=OFFLINE, location=122cddc95ec8,37369,1733206917509; forceNewPlan=false, retain=false 2024-12-03T06:22:00,740 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=afa4ab371a599cc21eb81082233626e3, ASSIGN 2024-12-03T06:22:00,741 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=afa4ab371a599cc21eb81082233626e3, ASSIGN; state=OFFLINE, location=122cddc95ec8,45577,1733206917466; forceNewPlan=false, retain=false 2024-12-03T06:22:00,743 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6badf4f9ef3213a92a5f6ace694c6482, ASSIGN 2024-12-03T06:22:00,744 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6badf4f9ef3213a92a5f6ace694c6482, ASSIGN; state=OFFLINE, location=122cddc95ec8,43629,1733206917367; forceNewPlan=false, retain=false 2024-12-03T06:22:00,746 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN 2024-12-03T06:22:00,748 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN; state=OFFLINE, location=122cddc95ec8,37369,1733206917509; forceNewPlan=false, retain=false 2024-12-03T06:22:00,750 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a6fcc2e40df410122347a4873dfd1129, ASSIGN 2024-12-03T06:22:00,751 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a6fcc2e40df410122347a4873dfd1129, ASSIGN; state=OFFLINE, location=122cddc95ec8,45577,1733206917466; forceNewPlan=false, retain=false 2024-12-03T06:22:00,753 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b5f7ab205e6c897503e3ec729aaa3170, ASSIGN 2024-12-03T06:22:00,754 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b5f7ab205e6c897503e3ec729aaa3170, ASSIGN; state=OFFLINE, location=122cddc95ec8,43629,1733206917367; forceNewPlan=false, retain=false 2024-12-03T06:22:00,755 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=050ab97be4cc9f90a155d3b261335a0d, ASSIGN 2024-12-03T06:22:00,757 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=050ab97be4cc9f90a155d3b261335a0d, ASSIGN; state=OFFLINE, location=122cddc95ec8,45577,1733206917466; forceNewPlan=false, retain=false 2024-12-03T06:22:00,758 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:00,759 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN; state=OFFLINE, location=122cddc95ec8,37369,1733206917509; forceNewPlan=false, retain=false 2024-12-03T06:22:00,761 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f495460f3e799a3e64378f8879c721bb, ASSIGN 2024-12-03T06:22:00,763 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f495460f3e799a3e64378f8879c721bb, ASSIGN; state=OFFLINE, location=122cddc95ec8,45577,1733206917466; forceNewPlan=false, retain=false 2024-12-03T06:22:00,765 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3c9e860d892b47160f56d24cb24e6010, ASSIGN 2024-12-03T06:22:00,766 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3c9e860d892b47160f56d24cb24e6010, ASSIGN; state=OFFLINE, location=122cddc95ec8,43629,1733206917367; forceNewPlan=false, retain=false 2024-12-03T06:22:00,768 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN 2024-12-03T06:22:00,769 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN; state=OFFLINE, location=122cddc95ec8,37369,1733206917509; forceNewPlan=false, retain=false 2024-12-03T06:22:00,771 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7d719355720f98b8bc93f39a6130075b, ASSIGN 2024-12-03T06:22:00,772 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7d719355720f98b8bc93f39a6130075b, ASSIGN; state=OFFLINE, location=122cddc95ec8,45577,1733206917466; forceNewPlan=false, retain=false 2024-12-03T06:22:00,773 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ca916e60e459eeefe786a877b8700115, ASSIGN 2024-12-03T06:22:00,775 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ca916e60e459eeefe786a877b8700115, ASSIGN; state=OFFLINE, location=122cddc95ec8,43629,1733206917367; forceNewPlan=false, retain=false 2024-12-03T06:22:00,776 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN 2024-12-03T06:22:00,777 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN; state=OFFLINE, location=122cddc95ec8,37369,1733206917509; forceNewPlan=false, retain=false 2024-12-03T06:22:00,779 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e884340ece2e3eb17a321ed6d089e5e7, ASSIGN 2024-12-03T06:22:00,780 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e884340ece2e3eb17a321ed6d089e5e7, ASSIGN; state=OFFLINE, location=122cddc95ec8,45577,1733206917466; forceNewPlan=false, retain=false 2024-12-03T06:22:00,782 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=33124cac09cf3a2502ce5e473bf614f4, ASSIGN 2024-12-03T06:22:00,783 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=33124cac09cf3a2502ce5e473bf614f4, ASSIGN; state=OFFLINE, location=122cddc95ec8,43629,1733206917367; forceNewPlan=false, retain=false 2024-12-03T06:22:00,785 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3bbc9d78d890c8d774a80646d9983ed6, ASSIGN 2024-12-03T06:22:00,786 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3bbc9d78d890c8d774a80646d9983ed6, ASSIGN; state=OFFLINE, location=122cddc95ec8,45577,1733206917466; forceNewPlan=false, retain=false 2024-12-03T06:22:00,788 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=40b4b0967914401ebc70c24d4b56a381, ASSIGN 2024-12-03T06:22:00,789 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=40b4b0967914401ebc70c24d4b56a381, ASSIGN; state=OFFLINE, location=122cddc95ec8,43629,1733206917367; forceNewPlan=false, retain=false 2024-12-03T06:22:00,791 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN 2024-12-03T06:22:00,792 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN; state=OFFLINE, location=122cddc95ec8,37369,1733206917509; forceNewPlan=false, retain=false 2024-12-03T06:22:00,868 INFO [122cddc95ec8:35025 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-12-03T06:22:00,869 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=6b148e6fa0896eea84e6c38d5293e44a, regionState=OPENING, regionLocation=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:00,870 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=e3e1e4a86d09a1acb1fcf9d92f129898, regionState=OPENING, regionLocation=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:00,871 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=afa4ab371a599cc21eb81082233626e3, regionState=OPENING, regionLocation=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:00,871 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=a6fcc2e40df410122347a4873dfd1129, regionState=OPENING, regionLocation=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:00,872 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=050ab97be4cc9f90a155d3b261335a0d, regionState=OPENING, regionLocation=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:00,872 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=f495460f3e799a3e64378f8879c721bb, regionState=OPENING, regionLocation=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:00,873 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=7d719355720f98b8bc93f39a6130075b, regionState=OPENING, regionLocation=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:00,873 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=e884340ece2e3eb17a321ed6d089e5e7, regionState=OPENING, regionLocation=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:00,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6b148e6fa0896eea84e6c38d5293e44a, ASSIGN because future has completed 2024-12-03T06:22:00,874 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3bbc9d78d890c8d774a80646d9983ed6, regionState=OPENING, regionLocation=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:00,874 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=671297fbd74543fd2b2deed06d08e6a7, regionState=OPENING, regionLocation=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:00,875 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=e9f6e863b0ed7a188cb723f7a298ba2c, regionState=OPENING, regionLocation=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:00,876 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6b148e6fa0896eea84e6c38d5293e44a, server=122cddc95ec8,45577,1733206917466}] 2024-12-03T06:22:00,876 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e3e1e4a86d09a1acb1fcf9d92f129898, ASSIGN because future has completed 2024-12-03T06:22:00,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=afa4ab371a599cc21eb81082233626e3, ASSIGN because future has completed 2024-12-03T06:22:00,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a6fcc2e40df410122347a4873dfd1129, ASSIGN because future has completed 2024-12-03T06:22:00,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=050ab97be4cc9f90a155d3b261335a0d, ASSIGN because future has completed 2024-12-03T06:22:00,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 050ab97be4cc9f90a155d3b261335a0d, server=122cddc95ec8,45577,1733206917466}] 2024-12-03T06:22:00,889 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f495460f3e799a3e64378f8879c721bb, ASSIGN because future has completed 2024-12-03T06:22:00,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7d719355720f98b8bc93f39a6130075b, ASSIGN because future has completed 2024-12-03T06:22:00,893 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e884340ece2e3eb17a321ed6d089e5e7, ASSIGN because future has completed 2024-12-03T06:22:00,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3bbc9d78d890c8d774a80646d9983ed6, ASSIGN because future has completed 2024-12-03T06:22:00,896 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3bbc9d78d890c8d774a80646d9983ed6, server=122cddc95ec8,45577,1733206917466}] 2024-12-03T06:22:00,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=671297fbd74543fd2b2deed06d08e6a7, ASSIGN because future has completed 2024-12-03T06:22:00,900 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e9f6e863b0ed7a188cb723f7a298ba2c, ASSIGN because future has completed 2024-12-03T06:22:00,902 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure e9f6e863b0ed7a188cb723f7a298ba2c, server=122cddc95ec8,43629,1733206917367}] 2024-12-03T06:22:00,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 671297fbd74543fd2b2deed06d08e6a7, server=122cddc95ec8,43629,1733206917367}] 2024-12-03T06:22:00,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure e884340ece2e3eb17a321ed6d089e5e7, server=122cddc95ec8,45577,1733206917466}] 2024-12-03T06:22:00,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7d719355720f98b8bc93f39a6130075b, server=122cddc95ec8,45577,1733206917466}] 2024-12-03T06:22:00,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure f495460f3e799a3e64378f8879c721bb, server=122cddc95ec8,45577,1733206917466}] 2024-12-03T06:22:00,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure a6fcc2e40df410122347a4873dfd1129, server=122cddc95ec8,45577,1733206917466}] 2024-12-03T06:22:00,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure afa4ab371a599cc21eb81082233626e3, server=122cddc95ec8,45577,1733206917466}] 2024-12-03T06:22:00,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure e3e1e4a86d09a1acb1fcf9d92f129898, server=122cddc95ec8,45577,1733206917466}] 2024-12-03T06:22:00,947 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=1e061778c0efa365f63836d0691ff7a8, regionState=OPENING, regionLocation=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:00,948 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=6badf4f9ef3213a92a5f6ace694c6482, regionState=OPENING, regionLocation=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:00,948 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=b5f7ab205e6c897503e3ec729aaa3170, regionState=OPENING, regionLocation=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:00,949 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=3c9e860d892b47160f56d24cb24e6010, regionState=OPENING, regionLocation=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:00,950 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=ca916e60e459eeefe786a877b8700115, regionState=OPENING, regionLocation=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:00,951 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=33124cac09cf3a2502ce5e473bf614f4, regionState=OPENING, regionLocation=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:00,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1e061778c0efa365f63836d0691ff7a8, ASSIGN because future has completed 2024-12-03T06:22:00,952 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=40b4b0967914401ebc70c24d4b56a381, regionState=OPENING, regionLocation=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:00,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e061778c0efa365f63836d0691ff7a8, server=122cddc95ec8,43629,1733206917367}] 2024-12-03T06:22:00,954 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6badf4f9ef3213a92a5f6ace694c6482, ASSIGN because future has completed 2024-12-03T06:22:00,956 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b5f7ab205e6c897503e3ec729aaa3170, ASSIGN because future has completed 2024-12-03T06:22:00,957 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3c9e860d892b47160f56d24cb24e6010, ASSIGN because future has completed 2024-12-03T06:22:00,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c9e860d892b47160f56d24cb24e6010, server=122cddc95ec8,43629,1733206917367}] 2024-12-03T06:22:00,959 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ca916e60e459eeefe786a877b8700115, ASSIGN because future has completed 2024-12-03T06:22:00,961 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=33124cac09cf3a2502ce5e473bf614f4, ASSIGN because future has completed 2024-12-03T06:22:00,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=40b4b0967914401ebc70c24d4b56a381, ASSIGN because future has completed 2024-12-03T06:22:00,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40b4b0967914401ebc70c24d4b56a381, server=122cddc95ec8,43629,1733206917367}] 2024-12-03T06:22:00,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 33124cac09cf3a2502ce5e473bf614f4, server=122cddc95ec8,43629,1733206917367}] 2024-12-03T06:22:00,972 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca916e60e459eeefe786a877b8700115, server=122cddc95ec8,43629,1733206917367}] 2024-12-03T06:22:00,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure b5f7ab205e6c897503e3ec729aaa3170, server=122cddc95ec8,43629,1733206917367}] 2024-12-03T06:22:00,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6badf4f9ef3213a92a5f6ace694c6482, server=122cddc95ec8,43629,1733206917367}] 2024-12-03T06:22:00,984 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=ae2330e378b6ad4b1a3e345529a0b27f, regionState=OPENING, regionLocation=122cddc95ec8,37369,1733206917509 2024-12-03T06:22:00,985 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=923ece5c25680df02ff5f922d838bc62, regionState=OPENING, regionLocation=122cddc95ec8,37369,1733206917509 2024-12-03T06:22:00,986 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=4e8636a0790f504025df15b4adf65735, regionState=OPENING, regionLocation=122cddc95ec8,37369,1733206917509 2024-12-03T06:22:00,987 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=07dbe58fce62a9e98b77016cec645f5e, regionState=OPENING, regionLocation=122cddc95ec8,37369,1733206917509 2024-12-03T06:22:00,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN because future has completed 2024-12-03T06:22:00,988 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1ddf2a494d8d0fcd31eac685512f5644, regionState=OPENING, regionLocation=122cddc95ec8,37369,1733206917509 2024-12-03T06:22:00,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae2330e378b6ad4b1a3e345529a0b27f, server=122cddc95ec8,37369,1733206917509}] 2024-12-03T06:22:00,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN because future has completed 2024-12-03T06:22:00,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN because future has completed 2024-12-03T06:22:00,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN because future has completed 2024-12-03T06:22:00,993 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN because future has completed 2024-12-03T06:22:00,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ddf2a494d8d0fcd31eac685512f5644, server=122cddc95ec8,37369,1733206917509}] 2024-12-03T06:22:00,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07dbe58fce62a9e98b77016cec645f5e, server=122cddc95ec8,37369,1733206917509}] 2024-12-03T06:22:01,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e8636a0790f504025df15b4adf65735, server=122cddc95ec8,37369,1733206917509}] 2024-12-03T06:22:01,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 923ece5c25680df02ff5f922d838bc62, server=122cddc95ec8,37369,1733206917509}] 2024-12-03T06:22:01,010 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=a332f3da50b1f1fe35aa4e0307fe35f8, regionState=OPENING, regionLocation=122cddc95ec8,37369,1733206917509 2024-12-03T06:22:01,011 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=e48f9dd0d4bf7a7f5e5fb4dac950f301, regionState=OPENING, regionLocation=122cddc95ec8,37369,1733206917509 2024-12-03T06:22:01,011 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=663ef4d3dbbcf7b1bb25bba1055da980, regionState=OPENING, regionLocation=122cddc95ec8,37369,1733206917509 2024-12-03T06:22:01,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN because future has completed 2024-12-03T06:22:01,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure a332f3da50b1f1fe35aa4e0307fe35f8, server=122cddc95ec8,37369,1733206917509}] 2024-12-03T06:22:01,014 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN because future has completed 2024-12-03T06:22:01,016 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN because future has completed 2024-12-03T06:22:01,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 663ef4d3dbbcf7b1bb25bba1055da980, server=122cddc95ec8,37369,1733206917509}] 2024-12-03T06:22:01,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,37369,1733206917509}] 2024-12-03T06:22:01,032 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T06:22:01,034 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54991, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T06:22:01,040 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,mmm,1733206919969.050ab97be4cc9f90a155d3b261335a0d. 2024-12-03T06:22:01,040 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 050ab97be4cc9f90a155d3b261335a0d, NAME => 'test,mmm,1733206919969.050ab97be4cc9f90a155d3b261335a0d.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-12-03T06:22:01,041 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 050ab97be4cc9f90a155d3b261335a0d 2024-12-03T06:22:01,041 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,mmm,1733206919969.050ab97be4cc9f90a155d3b261335a0d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:01,041 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 050ab97be4cc9f90a155d3b261335a0d 2024-12-03T06:22:01,041 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 050ab97be4cc9f90a155d3b261335a0d 2024-12-03T06:22:01,043 INFO [StoreOpener-050ab97be4cc9f90a155d3b261335a0d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 050ab97be4cc9f90a155d3b261335a0d 2024-12-03T06:22:01,045 INFO [StoreOpener-050ab97be4cc9f90a155d3b261335a0d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 050ab97be4cc9f90a155d3b261335a0d columnFamilyName family 2024-12-03T06:22:01,045 DEBUG [StoreOpener-050ab97be4cc9f90a155d3b261335a0d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:01,046 INFO [StoreOpener-050ab97be4cc9f90a155d3b261335a0d-1 {}] regionserver.HStore(327): Store=050ab97be4cc9f90a155d3b261335a0d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:01,046 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 050ab97be4cc9f90a155d3b261335a0d 2024-12-03T06:22:01,047 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/050ab97be4cc9f90a155d3b261335a0d 2024-12-03T06:22:01,048 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/050ab97be4cc9f90a155d3b261335a0d 2024-12-03T06:22:01,049 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 050ab97be4cc9f90a155d3b261335a0d 2024-12-03T06:22:01,049 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 050ab97be4cc9f90a155d3b261335a0d 2024-12-03T06:22:01,051 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 050ab97be4cc9f90a155d3b261335a0d 2024-12-03T06:22:01,053 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/050ab97be4cc9f90a155d3b261335a0d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:22:01,054 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 050ab97be4cc9f90a155d3b261335a0d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67034352, jitterRate=-0.0011103153228759766}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:01,054 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 050ab97be4cc9f90a155d3b261335a0d 2024-12-03T06:22:01,055 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T06:22:01,055 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 050ab97be4cc9f90a155d3b261335a0d: Running coprocessor pre-open hook at 1733206921041Writing region info on filesystem at 1733206921041Initializing all the Stores at 1733206921043 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206921043Cleaning up temporary data from old regions at 1733206921049 (+6 ms)Running coprocessor post-open hooks at 1733206921054 (+5 ms)Region opened successfully at 1733206921055 (+1 ms) 2024-12-03T06:22:01,057 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1733206919969.050ab97be4cc9f90a155d3b261335a0d., pid=32, masterSystemTime=1733206921031 2024-12-03T06:22:01,057 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54313, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T06:22:01,060 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1733206919969.050ab97be4cc9f90a155d3b261335a0d. 2024-12-03T06:22:01,060 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,mmm,1733206919969.050ab97be4cc9f90a155d3b261335a0d. 2024-12-03T06:22:01,061 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,kkk,1733206919969.f495460f3e799a3e64378f8879c721bb. 2024-12-03T06:22:01,061 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => f495460f3e799a3e64378f8879c721bb, NAME => 'test,kkk,1733206919969.f495460f3e799a3e64378f8879c721bb.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-12-03T06:22:01,061 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f495460f3e799a3e64378f8879c721bb 2024-12-03T06:22:01,061 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=050ab97be4cc9f90a155d3b261335a0d, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:01,062 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,kkk,1733206919969.f495460f3e799a3e64378f8879c721bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:01,062 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for f495460f3e799a3e64378f8879c721bb 2024-12-03T06:22:01,062 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for f495460f3e799a3e64378f8879c721bb 2024-12-03T06:22:01,065 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,ggg,1733206919969.ca916e60e459eeefe786a877b8700115. 2024-12-03T06:22:01,065 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => ca916e60e459eeefe786a877b8700115, NAME => 'test,ggg,1733206919969.ca916e60e459eeefe786a877b8700115.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-12-03T06:22:01,065 INFO [StoreOpener-f495460f3e799a3e64378f8879c721bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f495460f3e799a3e64378f8879c721bb 2024-12-03T06:22:01,066 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ca916e60e459eeefe786a877b8700115 2024-12-03T06:22:01,066 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,ggg,1733206919969.ca916e60e459eeefe786a877b8700115.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:01,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 050ab97be4cc9f90a155d3b261335a0d, server=122cddc95ec8,45577,1733206917466 because future has completed 2024-12-03T06:22:01,066 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for ca916e60e459eeefe786a877b8700115 2024-12-03T06:22:01,066 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for ca916e60e459eeefe786a877b8700115 2024-12-03T06:22:01,068 INFO [StoreOpener-f495460f3e799a3e64378f8879c721bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f495460f3e799a3e64378f8879c721bb columnFamilyName family 2024-12-03T06:22:01,068 DEBUG [StoreOpener-f495460f3e799a3e64378f8879c721bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:01,068 INFO [StoreOpener-ca916e60e459eeefe786a877b8700115-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ca916e60e459eeefe786a877b8700115 2024-12-03T06:22:01,069 INFO [StoreOpener-f495460f3e799a3e64378f8879c721bb-1 {}] regionserver.HStore(327): Store=f495460f3e799a3e64378f8879c721bb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:01,069 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for f495460f3e799a3e64378f8879c721bb 2024-12-03T06:22:01,070 INFO [StoreOpener-ca916e60e459eeefe786a877b8700115-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca916e60e459eeefe786a877b8700115 columnFamilyName family 2024-12-03T06:22:01,070 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/f495460f3e799a3e64378f8879c721bb 2024-12-03T06:22:01,070 DEBUG [StoreOpener-ca916e60e459eeefe786a877b8700115-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:01,071 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/f495460f3e799a3e64378f8879c721bb 2024-12-03T06:22:01,071 INFO [StoreOpener-ca916e60e459eeefe786a877b8700115-1 {}] regionserver.HStore(327): Store=ca916e60e459eeefe786a877b8700115/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:01,071 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for ca916e60e459eeefe786a877b8700115 2024-12-03T06:22:01,072 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for f495460f3e799a3e64378f8879c721bb 2024-12-03T06:22:01,072 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for f495460f3e799a3e64378f8879c721bb 2024-12-03T06:22:01,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=17 2024-12-03T06:22:01,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 050ab97be4cc9f90a155d3b261335a0d, server=122cddc95ec8,45577,1733206917466 in 178 msec 2024-12-03T06:22:01,073 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/ca916e60e459eeefe786a877b8700115 2024-12-03T06:22:01,073 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/ca916e60e459eeefe786a877b8700115 2024-12-03T06:22:01,074 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for ca916e60e459eeefe786a877b8700115 2024-12-03T06:22:01,074 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for ca916e60e459eeefe786a877b8700115 2024-12-03T06:22:01,075 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for f495460f3e799a3e64378f8879c721bb 2024-12-03T06:22:01,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=050ab97be4cc9f90a155d3b261335a0d, ASSIGN in 366 msec 2024-12-03T06:22:01,077 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for ca916e60e459eeefe786a877b8700115 2024-12-03T06:22:01,078 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/f495460f3e799a3e64378f8879c721bb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:22:01,078 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened f495460f3e799a3e64378f8879c721bb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62608251, jitterRate=-0.06706435978412628}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:01,078 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f495460f3e799a3e64378f8879c721bb 2024-12-03T06:22:01,079 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for f495460f3e799a3e64378f8879c721bb: Running coprocessor pre-open hook at 1733206921062Writing region info on filesystem at 1733206921062Initializing all the Stores at 1733206921064 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206921065 (+1 ms)Cleaning up temporary data from old regions at 1733206921072 (+7 ms)Running coprocessor post-open hooks at 1733206921078 (+6 ms)Region opened successfully at 1733206921078 2024-12-03T06:22:01,080 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1733206919969.f495460f3e799a3e64378f8879c721bb., pid=38, masterSystemTime=1733206921031 2024-12-03T06:22:01,080 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/ca916e60e459eeefe786a877b8700115/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:22:01,081 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened ca916e60e459eeefe786a877b8700115; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71644029, jitterRate=0.0675792247056961}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:01,081 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ca916e60e459eeefe786a877b8700115 2024-12-03T06:22:01,081 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for ca916e60e459eeefe786a877b8700115: Running coprocessor pre-open hook at 1733206921066Writing region info on filesystem at 1733206921066Initializing all the Stores at 1733206921067 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206921067Cleaning up temporary data from old regions at 1733206921074 (+7 ms)Running coprocessor post-open hooks at 1733206921081 (+7 ms)Region opened successfully at 1733206921081 2024-12-03T06:22:01,082 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1733206919969.ca916e60e459eeefe786a877b8700115., pid=46, masterSystemTime=1733206921055 2024-12-03T06:22:01,083 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1733206919969.f495460f3e799a3e64378f8879c721bb. 2024-12-03T06:22:01,083 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,kkk,1733206919969.f495460f3e799a3e64378f8879c721bb. 2024-12-03T06:22:01,083 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,ccc,1733206919969.3bbc9d78d890c8d774a80646d9983ed6. 2024-12-03T06:22:01,083 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 3bbc9d78d890c8d774a80646d9983ed6, NAME => 'test,ccc,1733206919969.3bbc9d78d890c8d774a80646d9983ed6.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-12-03T06:22:01,084 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=f495460f3e799a3e64378f8879c721bb, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:01,084 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3bbc9d78d890c8d774a80646d9983ed6 2024-12-03T06:22:01,084 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,ccc,1733206919969.3bbc9d78d890c8d774a80646d9983ed6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:01,084 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 3bbc9d78d890c8d774a80646d9983ed6 2024-12-03T06:22:01,084 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 3bbc9d78d890c8d774a80646d9983ed6 2024-12-03T06:22:01,086 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1733206919969.ca916e60e459eeefe786a877b8700115. 2024-12-03T06:22:01,086 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,ggg,1733206919969.ca916e60e459eeefe786a877b8700115. 2024-12-03T06:22:01,086 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,qqq,1733206919969.6badf4f9ef3213a92a5f6ace694c6482. 2024-12-03T06:22:01,086 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 6badf4f9ef3213a92a5f6ace694c6482, NAME => 'test,qqq,1733206919969.6badf4f9ef3213a92a5f6ace694c6482.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-12-03T06:22:01,087 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6badf4f9ef3213a92a5f6ace694c6482 2024-12-03T06:22:01,087 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=ca916e60e459eeefe786a877b8700115, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:01,087 INFO [StoreOpener-3bbc9d78d890c8d774a80646d9983ed6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3bbc9d78d890c8d774a80646d9983ed6 2024-12-03T06:22:01,087 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,qqq,1733206919969.6badf4f9ef3213a92a5f6ace694c6482.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:01,087 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure f495460f3e799a3e64378f8879c721bb, server=122cddc95ec8,45577,1733206917466 because future has completed 2024-12-03T06:22:01,087 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 6badf4f9ef3213a92a5f6ace694c6482 2024-12-03T06:22:01,087 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 6badf4f9ef3213a92a5f6ace694c6482 2024-12-03T06:22:01,089 INFO [StoreOpener-6badf4f9ef3213a92a5f6ace694c6482-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6badf4f9ef3213a92a5f6ace694c6482 2024-12-03T06:22:01,090 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca916e60e459eeefe786a877b8700115, server=122cddc95ec8,43629,1733206917367 because future has completed 2024-12-03T06:22:01,091 INFO [StoreOpener-3bbc9d78d890c8d774a80646d9983ed6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3bbc9d78d890c8d774a80646d9983ed6 columnFamilyName family 2024-12-03T06:22:01,092 DEBUG [StoreOpener-3bbc9d78d890c8d774a80646d9983ed6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:01,092 INFO [StoreOpener-6badf4f9ef3213a92a5f6ace694c6482-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6badf4f9ef3213a92a5f6ace694c6482 columnFamilyName family 2024-12-03T06:22:01,092 DEBUG [StoreOpener-6badf4f9ef3213a92a5f6ace694c6482-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:01,092 INFO [StoreOpener-3bbc9d78d890c8d774a80646d9983ed6-1 {}] regionserver.HStore(327): Store=3bbc9d78d890c8d774a80646d9983ed6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:01,093 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 3bbc9d78d890c8d774a80646d9983ed6 2024-12-03T06:22:01,093 INFO [StoreOpener-6badf4f9ef3213a92a5f6ace694c6482-1 {}] regionserver.HStore(327): Store=6badf4f9ef3213a92a5f6ace694c6482/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:01,093 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 6badf4f9ef3213a92a5f6ace694c6482 2024-12-03T06:22:01,094 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/3bbc9d78d890c8d774a80646d9983ed6 2024-12-03T06:22:01,094 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/6badf4f9ef3213a92a5f6ace694c6482 2024-12-03T06:22:01,095 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=15 2024-12-03T06:22:01,095 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/3bbc9d78d890c8d774a80646d9983ed6 2024-12-03T06:22:01,095 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure f495460f3e799a3e64378f8879c721bb, server=122cddc95ec8,45577,1733206917466 in 170 msec 2024-12-03T06:22:01,095 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/6badf4f9ef3213a92a5f6ace694c6482 2024-12-03T06:22:01,095 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 3bbc9d78d890c8d774a80646d9983ed6 2024-12-03T06:22:01,095 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 6badf4f9ef3213a92a5f6ace694c6482 2024-12-03T06:22:01,095 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 3bbc9d78d890c8d774a80646d9983ed6 2024-12-03T06:22:01,095 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 6badf4f9ef3213a92a5f6ace694c6482 2024-12-03T06:22:01,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f495460f3e799a3e64378f8879c721bb, ASSIGN in 389 msec 2024-12-03T06:22:01,098 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 6badf4f9ef3213a92a5f6ace694c6482 2024-12-03T06:22:01,098 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 3bbc9d78d890c8d774a80646d9983ed6 2024-12-03T06:22:01,102 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/6badf4f9ef3213a92a5f6ace694c6482/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:22:01,102 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/3bbc9d78d890c8d774a80646d9983ed6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:22:01,102 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 6badf4f9ef3213a92a5f6ace694c6482; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75257516, jitterRate=0.12142437696456909}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:01,103 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 3bbc9d78d890c8d774a80646d9983ed6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69062240, jitterRate=0.02910757064819336}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:01,103 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6badf4f9ef3213a92a5f6ace694c6482 2024-12-03T06:22:01,103 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3bbc9d78d890c8d774a80646d9983ed6 2024-12-03T06:22:01,103 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 6badf4f9ef3213a92a5f6ace694c6482: Running coprocessor pre-open hook at 1733206921087Writing region info on filesystem at 1733206921087Initializing all the Stores at 1733206921089 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206921089Cleaning up temporary data from old regions at 1733206921095 (+6 ms)Running coprocessor post-open hooks at 1733206921103 (+8 ms)Region opened successfully at 1733206921103 2024-12-03T06:22:01,103 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 3bbc9d78d890c8d774a80646d9983ed6: Running coprocessor pre-open hook at 1733206921084Writing region info on filesystem at 1733206921084Initializing all the Stores at 1733206921086 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206921086Cleaning up temporary data from old regions at 1733206921095 (+9 ms)Running coprocessor post-open hooks at 1733206921103 (+8 ms)Region opened successfully at 1733206921103 2024-12-03T06:22:01,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=11 2024-12-03T06:22:01,104 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1733206919969.6badf4f9ef3213a92a5f6ace694c6482., pid=48, masterSystemTime=1733206921055 2024-12-03T06:22:01,104 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1733206919969.3bbc9d78d890c8d774a80646d9983ed6., pid=33, masterSystemTime=1733206921031 2024-12-03T06:22:01,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure ca916e60e459eeefe786a877b8700115, server=122cddc95ec8,43629,1733206917367 in 127 msec 2024-12-03T06:22:01,107 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1733206919969.6badf4f9ef3213a92a5f6ace694c6482. 2024-12-03T06:22:01,107 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,qqq,1733206919969.6badf4f9ef3213a92a5f6ace694c6482. 2024-12-03T06:22:01,107 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,bbb,1733206919969.40b4b0967914401ebc70c24d4b56a381. 2024-12-03T06:22:01,108 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 40b4b0967914401ebc70c24d4b56a381, NAME => 'test,bbb,1733206919969.40b4b0967914401ebc70c24d4b56a381.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-12-03T06:22:01,108 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 40b4b0967914401ebc70c24d4b56a381 2024-12-03T06:22:01,108 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,bbb,1733206919969.40b4b0967914401ebc70c24d4b56a381.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:01,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ca916e60e459eeefe786a877b8700115, ASSIGN in 398 msec 2024-12-03T06:22:01,108 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 40b4b0967914401ebc70c24d4b56a381 2024-12-03T06:22:01,108 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 40b4b0967914401ebc70c24d4b56a381 2024-12-03T06:22:01,109 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=6badf4f9ef3213a92a5f6ace694c6482, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:01,110 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1733206919969.3bbc9d78d890c8d774a80646d9983ed6. 2024-12-03T06:22:01,110 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,ccc,1733206919969.3bbc9d78d890c8d774a80646d9983ed6. 2024-12-03T06:22:01,110 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,ooo,1733206919969.a6fcc2e40df410122347a4873dfd1129. 2024-12-03T06:22:01,110 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => a6fcc2e40df410122347a4873dfd1129, NAME => 'test,ooo,1733206919969.a6fcc2e40df410122347a4873dfd1129.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-12-03T06:22:01,111 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a6fcc2e40df410122347a4873dfd1129 2024-12-03T06:22:01,111 INFO [StoreOpener-40b4b0967914401ebc70c24d4b56a381-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 40b4b0967914401ebc70c24d4b56a381 2024-12-03T06:22:01,111 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,ooo,1733206919969.a6fcc2e40df410122347a4873dfd1129.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:01,111 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for a6fcc2e40df410122347a4873dfd1129 2024-12-03T06:22:01,111 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for a6fcc2e40df410122347a4873dfd1129 2024-12-03T06:22:01,113 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6badf4f9ef3213a92a5f6ace694c6482, server=122cddc95ec8,43629,1733206917367 because future has completed 2024-12-03T06:22:01,113 INFO [StoreOpener-a6fcc2e40df410122347a4873dfd1129-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a6fcc2e40df410122347a4873dfd1129 2024-12-03T06:22:01,113 INFO [StoreOpener-40b4b0967914401ebc70c24d4b56a381-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40b4b0967914401ebc70c24d4b56a381 columnFamilyName family 2024-12-03T06:22:01,114 DEBUG [StoreOpener-40b4b0967914401ebc70c24d4b56a381-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:01,114 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3bbc9d78d890c8d774a80646d9983ed6, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:01,114 INFO [StoreOpener-40b4b0967914401ebc70c24d4b56a381-1 {}] regionserver.HStore(327): Store=40b4b0967914401ebc70c24d4b56a381/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:01,114 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 40b4b0967914401ebc70c24d4b56a381 2024-12-03T06:22:01,116 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/40b4b0967914401ebc70c24d4b56a381 2024-12-03T06:22:01,117 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/40b4b0967914401ebc70c24d4b56a381 2024-12-03T06:22:01,118 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 40b4b0967914401ebc70c24d4b56a381 2024-12-03T06:22:01,118 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3bbc9d78d890c8d774a80646d9983ed6, server=122cddc95ec8,45577,1733206917466 because future has completed 2024-12-03T06:22:01,118 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 40b4b0967914401ebc70c24d4b56a381 2024-12-03T06:22:01,121 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 40b4b0967914401ebc70c24d4b56a381 2024-12-03T06:22:01,122 INFO [StoreOpener-a6fcc2e40df410122347a4873dfd1129-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6fcc2e40df410122347a4873dfd1129 columnFamilyName family 2024-12-03T06:22:01,122 DEBUG [StoreOpener-a6fcc2e40df410122347a4873dfd1129-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:01,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=21 2024-12-03T06:22:01,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 6badf4f9ef3213a92a5f6ace694c6482, server=122cddc95ec8,43629,1733206917367 in 139 msec 2024-12-03T06:22:01,124 INFO [StoreOpener-a6fcc2e40df410122347a4873dfd1129-1 {}] regionserver.HStore(327): Store=a6fcc2e40df410122347a4873dfd1129/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:01,124 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for a6fcc2e40df410122347a4873dfd1129 2024-12-03T06:22:01,126 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/a6fcc2e40df410122347a4873dfd1129 2024-12-03T06:22:01,126 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/40b4b0967914401ebc70c24d4b56a381/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:22:01,127 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/a6fcc2e40df410122347a4873dfd1129 2024-12-03T06:22:01,127 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 40b4b0967914401ebc70c24d4b56a381; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69689256, jitterRate=0.03845083713531494}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:01,127 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 40b4b0967914401ebc70c24d4b56a381 2024-12-03T06:22:01,128 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 40b4b0967914401ebc70c24d4b56a381: Running coprocessor pre-open hook at 1733206921108Writing region info on filesystem at 1733206921108Initializing all the Stores at 1733206921110 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206921110Cleaning up temporary data from old regions at 1733206921118 (+8 ms)Running coprocessor post-open hooks at 1733206921127 (+9 ms)Region opened successfully at 1733206921127 2024-12-03T06:22:01,128 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for a6fcc2e40df410122347a4873dfd1129 2024-12-03T06:22:01,128 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for a6fcc2e40df410122347a4873dfd1129 2024-12-03T06:22:01,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6badf4f9ef3213a92a5f6ace694c6482, ASSIGN in 418 msec 2024-12-03T06:22:01,129 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1733206919969.40b4b0967914401ebc70c24d4b56a381., pid=44, masterSystemTime=1733206921055 2024-12-03T06:22:01,132 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for a6fcc2e40df410122347a4873dfd1129 2024-12-03T06:22:01,133 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1733206919969.40b4b0967914401ebc70c24d4b56a381. 2024-12-03T06:22:01,133 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,bbb,1733206919969.40b4b0967914401ebc70c24d4b56a381. 2024-12-03T06:22:01,134 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,zzz,1733206919969.671297fbd74543fd2b2deed06d08e6a7. 2024-12-03T06:22:01,134 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 671297fbd74543fd2b2deed06d08e6a7, NAME => 'test,zzz,1733206919969.671297fbd74543fd2b2deed06d08e6a7.', STARTKEY => 'zzz', ENDKEY => ''} 2024-12-03T06:22:01,134 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 671297fbd74543fd2b2deed06d08e6a7 2024-12-03T06:22:01,134 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,zzz,1733206919969.671297fbd74543fd2b2deed06d08e6a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:01,134 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 671297fbd74543fd2b2deed06d08e6a7 2024-12-03T06:22:01,134 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 671297fbd74543fd2b2deed06d08e6a7 2024-12-03T06:22:01,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=7 2024-12-03T06:22:01,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 3bbc9d78d890c8d774a80646d9983ed6, server=122cddc95ec8,45577,1733206917466 in 234 msec 2024-12-03T06:22:01,137 INFO [StoreOpener-671297fbd74543fd2b2deed06d08e6a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 671297fbd74543fd2b2deed06d08e6a7 2024-12-03T06:22:01,138 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/a6fcc2e40df410122347a4873dfd1129/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:22:01,140 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened a6fcc2e40df410122347a4873dfd1129; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69463385, jitterRate=0.03508509695529938}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:01,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3bbc9d78d890c8d774a80646d9983ed6, ASSIGN in 430 msec 2024-12-03T06:22:01,140 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a6fcc2e40df410122347a4873dfd1129 2024-12-03T06:22:01,141 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for a6fcc2e40df410122347a4873dfd1129: Running coprocessor pre-open hook at 1733206921111Writing region info on filesystem at 1733206921111Initializing all the Stores at 1733206921113 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206921113Cleaning up temporary data from old regions at 1733206921128 (+15 ms)Running coprocessor post-open hooks at 1733206921141 (+13 ms)Region opened successfully at 1733206921141 2024-12-03T06:22:01,141 INFO [StoreOpener-671297fbd74543fd2b2deed06d08e6a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 671297fbd74543fd2b2deed06d08e6a7 columnFamilyName family 2024-12-03T06:22:01,142 DEBUG [StoreOpener-671297fbd74543fd2b2deed06d08e6a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:01,143 INFO [StoreOpener-671297fbd74543fd2b2deed06d08e6a7-1 {}] regionserver.HStore(327): Store=671297fbd74543fd2b2deed06d08e6a7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:01,143 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 671297fbd74543fd2b2deed06d08e6a7 2024-12-03T06:22:01,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T06:22:01,144 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1733206919969.a6fcc2e40df410122347a4873dfd1129., pid=39, masterSystemTime=1733206921031 2024-12-03T06:22:01,145 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=40b4b0967914401ebc70c24d4b56a381, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:01,148 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/671297fbd74543fd2b2deed06d08e6a7 2024-12-03T06:22:01,149 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40b4b0967914401ebc70c24d4b56a381, server=122cddc95ec8,43629,1733206917367 because future has completed 2024-12-03T06:22:01,150 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1733206919969.a6fcc2e40df410122347a4873dfd1129. 2024-12-03T06:22:01,150 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/671297fbd74543fd2b2deed06d08e6a7 2024-12-03T06:22:01,150 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,ooo,1733206919969.a6fcc2e40df410122347a4873dfd1129. 2024-12-03T06:22:01,150 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,uuu,1733206919969.e3e1e4a86d09a1acb1fcf9d92f129898. 2024-12-03T06:22:01,150 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => e3e1e4a86d09a1acb1fcf9d92f129898, NAME => 'test,uuu,1733206919969.e3e1e4a86d09a1acb1fcf9d92f129898.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-12-03T06:22:01,150 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=a6fcc2e40df410122347a4873dfd1129, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:01,151 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e3e1e4a86d09a1acb1fcf9d92f129898 2024-12-03T06:22:01,151 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,uuu,1733206919969.e3e1e4a86d09a1acb1fcf9d92f129898.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:01,151 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for e3e1e4a86d09a1acb1fcf9d92f129898 2024-12-03T06:22:01,151 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 671297fbd74543fd2b2deed06d08e6a7 2024-12-03T06:22:01,151 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for e3e1e4a86d09a1acb1fcf9d92f129898 2024-12-03T06:22:01,151 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 671297fbd74543fd2b2deed06d08e6a7 2024-12-03T06:22:01,151 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,sss,1733206919969.4e8636a0790f504025df15b4adf65735. 2024-12-03T06:22:01,152 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 4e8636a0790f504025df15b4adf65735, NAME => 'test,sss,1733206919969.4e8636a0790f504025df15b4adf65735.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-12-03T06:22:01,154 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4e8636a0790f504025df15b4adf65735 2024-12-03T06:22:01,154 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,sss,1733206919969.4e8636a0790f504025df15b4adf65735.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:01,154 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 4e8636a0790f504025df15b4adf65735 2024-12-03T06:22:01,155 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 4e8636a0790f504025df15b4adf65735 2024-12-03T06:22:01,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure a6fcc2e40df410122347a4873dfd1129, server=122cddc95ec8,45577,1733206917466 because future has completed 2024-12-03T06:22:01,156 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 671297fbd74543fd2b2deed06d08e6a7 2024-12-03T06:22:01,158 INFO [StoreOpener-e3e1e4a86d09a1acb1fcf9d92f129898-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e3e1e4a86d09a1acb1fcf9d92f129898 2024-12-03T06:22:01,160 INFO [StoreOpener-e3e1e4a86d09a1acb1fcf9d92f129898-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3e1e4a86d09a1acb1fcf9d92f129898 columnFamilyName family 2024-12-03T06:22:01,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=6 2024-12-03T06:22:01,160 DEBUG [StoreOpener-e3e1e4a86d09a1acb1fcf9d92f129898-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:01,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 40b4b0967914401ebc70c24d4b56a381, server=122cddc95ec8,43629,1733206917367 in 192 msec 2024-12-03T06:22:01,161 INFO [StoreOpener-e3e1e4a86d09a1acb1fcf9d92f129898-1 {}] regionserver.HStore(327): Store=e3e1e4a86d09a1acb1fcf9d92f129898/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:01,161 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for e3e1e4a86d09a1acb1fcf9d92f129898 2024-12-03T06:22:01,163 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/e3e1e4a86d09a1acb1fcf9d92f129898 2024-12-03T06:22:01,163 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=40b4b0967914401ebc70c24d4b56a381, ASSIGN in 455 msec 2024-12-03T06:22:01,163 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/e3e1e4a86d09a1acb1fcf9d92f129898 2024-12-03T06:22:01,164 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for e3e1e4a86d09a1acb1fcf9d92f129898 2024-12-03T06:22:01,164 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for e3e1e4a86d09a1acb1fcf9d92f129898 2024-12-03T06:22:01,166 INFO [StoreOpener-4e8636a0790f504025df15b4adf65735-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4e8636a0790f504025df15b4adf65735 2024-12-03T06:22:01,167 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for e3e1e4a86d09a1acb1fcf9d92f129898 2024-12-03T06:22:01,169 INFO [StoreOpener-4e8636a0790f504025df15b4adf65735-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e8636a0790f504025df15b4adf65735 columnFamilyName family 2024-12-03T06:22:01,169 DEBUG [StoreOpener-4e8636a0790f504025df15b4adf65735-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:01,170 INFO [StoreOpener-4e8636a0790f504025df15b4adf65735-1 {}] regionserver.HStore(327): Store=4e8636a0790f504025df15b4adf65735/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:01,170 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 4e8636a0790f504025df15b4adf65735 2024-12-03T06:22:01,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=19 2024-12-03T06:22:01,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure a6fcc2e40df410122347a4873dfd1129, server=122cddc95ec8,45577,1733206917466 in 239 msec 2024-12-03T06:22:01,171 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/4e8636a0790f504025df15b4adf65735 2024-12-03T06:22:01,172 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/4e8636a0790f504025df15b4adf65735 2024-12-03T06:22:01,172 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 4e8636a0790f504025df15b4adf65735 2024-12-03T06:22:01,172 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 4e8636a0790f504025df15b4adf65735 2024-12-03T06:22:01,175 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/671297fbd74543fd2b2deed06d08e6a7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:22:01,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a6fcc2e40df410122347a4873dfd1129, ASSIGN in 466 msec 2024-12-03T06:22:01,175 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/e3e1e4a86d09a1acb1fcf9d92f129898/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:22:01,176 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 671297fbd74543fd2b2deed06d08e6a7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69448160, jitterRate=0.03485822677612305}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:01,176 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 671297fbd74543fd2b2deed06d08e6a7 2024-12-03T06:22:01,176 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 671297fbd74543fd2b2deed06d08e6a7: Running coprocessor pre-open hook at 1733206921135Writing region info on filesystem at 1733206921135Initializing all the Stores at 1733206921137 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206921137Cleaning up temporary data from old regions at 1733206921151 (+14 ms)Running coprocessor post-open hooks at 1733206921176 (+25 ms)Region opened successfully at 1733206921176 2024-12-03T06:22:01,176 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened e3e1e4a86d09a1acb1fcf9d92f129898; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74428168, jitterRate=0.10906612873077393}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:01,176 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e3e1e4a86d09a1acb1fcf9d92f129898 2024-12-03T06:22:01,176 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 4e8636a0790f504025df15b4adf65735 2024-12-03T06:22:01,176 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for e3e1e4a86d09a1acb1fcf9d92f129898: Running coprocessor pre-open hook at 1733206921151Writing region info on filesystem at 1733206921151Initializing all the Stores at 1733206921154 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206921154Cleaning up temporary data from old regions at 1733206921164 (+10 ms)Running coprocessor post-open hooks at 1733206921176 (+12 ms)Region opened successfully at 1733206921176 2024-12-03T06:22:01,177 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1733206919969.671297fbd74543fd2b2deed06d08e6a7., pid=35, masterSystemTime=1733206921055 2024-12-03T06:22:01,177 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1733206919969.e3e1e4a86d09a1acb1fcf9d92f129898., pid=41, masterSystemTime=1733206921031 2024-12-03T06:22:01,180 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1733206919969.e3e1e4a86d09a1acb1fcf9d92f129898. 2024-12-03T06:22:01,181 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,uuu,1733206919969.e3e1e4a86d09a1acb1fcf9d92f129898. 2024-12-03T06:22:01,181 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,rrr,1733206919969.afa4ab371a599cc21eb81082233626e3. 2024-12-03T06:22:01,181 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => afa4ab371a599cc21eb81082233626e3, NAME => 'test,rrr,1733206919969.afa4ab371a599cc21eb81082233626e3.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-12-03T06:22:01,182 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test afa4ab371a599cc21eb81082233626e3 2024-12-03T06:22:01,182 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,rrr,1733206919969.afa4ab371a599cc21eb81082233626e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:01,182 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/4e8636a0790f504025df15b4adf65735/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:22:01,182 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for afa4ab371a599cc21eb81082233626e3 2024-12-03T06:22:01,182 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for afa4ab371a599cc21eb81082233626e3 2024-12-03T06:22:01,182 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1733206919969.671297fbd74543fd2b2deed06d08e6a7. 2024-12-03T06:22:01,182 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,zzz,1733206919969.671297fbd74543fd2b2deed06d08e6a7. 2024-12-03T06:22:01,182 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,ttt,1733206919969.1e061778c0efa365f63836d0691ff7a8. 2024-12-03T06:22:01,183 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 1e061778c0efa365f63836d0691ff7a8, NAME => 'test,ttt,1733206919969.1e061778c0efa365f63836d0691ff7a8.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-12-03T06:22:01,183 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 4e8636a0790f504025df15b4adf65735; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62342357, jitterRate=-0.07102648913860321}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:01,183 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4e8636a0790f504025df15b4adf65735 2024-12-03T06:22:01,183 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 4e8636a0790f504025df15b4adf65735: Running coprocessor pre-open hook at 1733206921155Writing region info on filesystem at 1733206921155Initializing all the Stores at 1733206921156 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206921157 (+1 ms)Cleaning up temporary data from old regions at 1733206921173 (+16 ms)Running coprocessor post-open hooks at 1733206921183 (+10 ms)Region opened successfully at 1733206921183 2024-12-03T06:22:01,183 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1e061778c0efa365f63836d0691ff7a8 2024-12-03T06:22:01,183 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,ttt,1733206919969.1e061778c0efa365f63836d0691ff7a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:01,183 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 1e061778c0efa365f63836d0691ff7a8 2024-12-03T06:22:01,183 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 1e061778c0efa365f63836d0691ff7a8 2024-12-03T06:22:01,185 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=e3e1e4a86d09a1acb1fcf9d92f129898, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:01,185 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1733206919969.4e8636a0790f504025df15b4adf65735., pid=52, masterSystemTime=1733206921141 2024-12-03T06:22:01,186 INFO [StoreOpener-afa4ab371a599cc21eb81082233626e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region afa4ab371a599cc21eb81082233626e3 2024-12-03T06:22:01,186 INFO [StoreOpener-1e061778c0efa365f63836d0691ff7a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1e061778c0efa365f63836d0691ff7a8 2024-12-03T06:22:01,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure e3e1e4a86d09a1acb1fcf9d92f129898, server=122cddc95ec8,45577,1733206917466 because future has completed 2024-12-03T06:22:01,189 INFO [StoreOpener-afa4ab371a599cc21eb81082233626e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region afa4ab371a599cc21eb81082233626e3 columnFamilyName family 2024-12-03T06:22:01,189 INFO [StoreOpener-1e061778c0efa365f63836d0691ff7a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e061778c0efa365f63836d0691ff7a8 columnFamilyName family 2024-12-03T06:22:01,189 DEBUG [StoreOpener-1e061778c0efa365f63836d0691ff7a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:01,189 DEBUG [StoreOpener-afa4ab371a599cc21eb81082233626e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:01,190 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=671297fbd74543fd2b2deed06d08e6a7, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:01,190 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1733206919969.4e8636a0790f504025df15b4adf65735. 2024-12-03T06:22:01,190 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,sss,1733206919969.4e8636a0790f504025df15b4adf65735. 2024-12-03T06:22:01,190 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,yyy,1733206919969.ae2330e378b6ad4b1a3e345529a0b27f. 2024-12-03T06:22:01,190 INFO [StoreOpener-afa4ab371a599cc21eb81082233626e3-1 {}] regionserver.HStore(327): Store=afa4ab371a599cc21eb81082233626e3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:01,190 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => ae2330e378b6ad4b1a3e345529a0b27f, NAME => 'test,yyy,1733206919969.ae2330e378b6ad4b1a3e345529a0b27f.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-12-03T06:22:01,191 INFO [StoreOpener-1e061778c0efa365f63836d0691ff7a8-1 {}] regionserver.HStore(327): Store=1e061778c0efa365f63836d0691ff7a8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:01,191 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for afa4ab371a599cc21eb81082233626e3 2024-12-03T06:22:01,191 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 1e061778c0efa365f63836d0691ff7a8 2024-12-03T06:22:01,191 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ae2330e378b6ad4b1a3e345529a0b27f 2024-12-03T06:22:01,191 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,yyy,1733206919969.ae2330e378b6ad4b1a3e345529a0b27f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:01,191 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for ae2330e378b6ad4b1a3e345529a0b27f 2024-12-03T06:22:01,191 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for ae2330e378b6ad4b1a3e345529a0b27f 2024-12-03T06:22:01,192 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/afa4ab371a599cc21eb81082233626e3 2024-12-03T06:22:01,193 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/1e061778c0efa365f63836d0691ff7a8 2024-12-03T06:22:01,193 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/afa4ab371a599cc21eb81082233626e3 2024-12-03T06:22:01,193 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/1e061778c0efa365f63836d0691ff7a8 2024-12-03T06:22:01,193 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for afa4ab371a599cc21eb81082233626e3 2024-12-03T06:22:01,193 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for afa4ab371a599cc21eb81082233626e3 2024-12-03T06:22:01,194 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 1e061778c0efa365f63836d0691ff7a8 2024-12-03T06:22:01,195 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 1e061778c0efa365f63836d0691ff7a8 2024-12-03T06:22:01,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 671297fbd74543fd2b2deed06d08e6a7, server=122cddc95ec8,43629,1733206917367 because future has completed 2024-12-03T06:22:01,195 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=4e8636a0790f504025df15b4adf65735, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,37369,1733206917509 2024-12-03T06:22:01,196 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for afa4ab371a599cc21eb81082233626e3 2024-12-03T06:22:01,198 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 1e061778c0efa365f63836d0691ff7a8 2024-12-03T06:22:01,199 INFO [StoreOpener-ae2330e378b6ad4b1a3e345529a0b27f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ae2330e378b6ad4b1a3e345529a0b27f 2024-12-03T06:22:01,200 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e8636a0790f504025df15b4adf65735, server=122cddc95ec8,37369,1733206917509 because future has completed 2024-12-03T06:22:01,202 INFO [StoreOpener-ae2330e378b6ad4b1a3e345529a0b27f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae2330e378b6ad4b1a3e345529a0b27f columnFamilyName family 2024-12-03T06:22:01,202 DEBUG [StoreOpener-ae2330e378b6ad4b1a3e345529a0b27f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:01,204 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/afa4ab371a599cc21eb81082233626e3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:22:01,204 INFO [StoreOpener-ae2330e378b6ad4b1a3e345529a0b27f-1 {}] regionserver.HStore(327): Store=ae2330e378b6ad4b1a3e345529a0b27f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:01,204 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for ae2330e378b6ad4b1a3e345529a0b27f 2024-12-03T06:22:01,205 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened afa4ab371a599cc21eb81082233626e3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72045462, jitterRate=0.07356104254722595}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:01,205 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for afa4ab371a599cc21eb81082233626e3 2024-12-03T06:22:01,206 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for afa4ab371a599cc21eb81082233626e3: Running coprocessor pre-open hook at 1733206921182Writing region info on filesystem at 1733206921182Initializing all the Stores at 1733206921185 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206921185Cleaning up temporary data from old regions at 1733206921194 (+9 ms)Running coprocessor post-open hooks at 1733206921205 (+11 ms)Region opened successfully at 1733206921205 2024-12-03T06:22:01,206 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/ae2330e378b6ad4b1a3e345529a0b27f 2024-12-03T06:22:01,206 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/ae2330e378b6ad4b1a3e345529a0b27f 2024-12-03T06:22:01,207 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for ae2330e378b6ad4b1a3e345529a0b27f 2024-12-03T06:22:01,207 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for ae2330e378b6ad4b1a3e345529a0b27f 2024-12-03T06:22:01,207 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1733206919969.afa4ab371a599cc21eb81082233626e3., pid=40, masterSystemTime=1733206921031 2024-12-03T06:22:01,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=30 2024-12-03T06:22:01,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 671297fbd74543fd2b2deed06d08e6a7, server=122cddc95ec8,43629,1733206917367 in 295 msec 2024-12-03T06:22:01,210 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1733206919969.afa4ab371a599cc21eb81082233626e3. 2024-12-03T06:22:01,210 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,rrr,1733206919969.afa4ab371a599cc21eb81082233626e3. 2024-12-03T06:22:01,211 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,xxx,1733206919969.6b148e6fa0896eea84e6c38d5293e44a. 2024-12-03T06:22:01,211 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for ae2330e378b6ad4b1a3e345529a0b27f 2024-12-03T06:22:01,211 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 6b148e6fa0896eea84e6c38d5293e44a, NAME => 'test,xxx,1733206919969.6b148e6fa0896eea84e6c38d5293e44a.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-12-03T06:22:01,211 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6b148e6fa0896eea84e6c38d5293e44a 2024-12-03T06:22:01,211 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,xxx,1733206919969.6b148e6fa0896eea84e6c38d5293e44a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:01,211 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 6b148e6fa0896eea84e6c38d5293e44a 2024-12-03T06:22:01,212 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 6b148e6fa0896eea84e6c38d5293e44a 2024-12-03T06:22:01,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=671297fbd74543fd2b2deed06d08e6a7, ASSIGN in 502 msec 2024-12-03T06:22:01,214 INFO [StoreOpener-6b148e6fa0896eea84e6c38d5293e44a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6b148e6fa0896eea84e6c38d5293e44a 2024-12-03T06:22:01,214 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/1e061778c0efa365f63836d0691ff7a8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:22:01,215 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=afa4ab371a599cc21eb81082233626e3, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:01,215 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/ae2330e378b6ad4b1a3e345529a0b27f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:22:01,215 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 1e061778c0efa365f63836d0691ff7a8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74988112, jitterRate=0.11740994453430176}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:01,215 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1e061778c0efa365f63836d0691ff7a8 2024-12-03T06:22:01,216 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 1e061778c0efa365f63836d0691ff7a8: Running coprocessor pre-open hook at 1733206921184Writing region info on filesystem at 1733206921184Initializing all the Stores at 1733206921185 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206921185Cleaning up temporary data from old regions at 1733206921195 (+10 ms)Running coprocessor post-open hooks at 1733206921216 (+21 ms)Region opened successfully at 1733206921216 2024-12-03T06:22:01,216 INFO [StoreOpener-6b148e6fa0896eea84e6c38d5293e44a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b148e6fa0896eea84e6c38d5293e44a columnFamilyName family 2024-12-03T06:22:01,216 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened ae2330e378b6ad4b1a3e345529a0b27f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75484813, jitterRate=0.12481136620044708}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:01,217 DEBUG [StoreOpener-6b148e6fa0896eea84e6c38d5293e44a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:01,217 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ae2330e378b6ad4b1a3e345529a0b27f 2024-12-03T06:22:01,217 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1733206919969.1e061778c0efa365f63836d0691ff7a8., pid=42, masterSystemTime=1733206921055 2024-12-03T06:22:01,217 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for ae2330e378b6ad4b1a3e345529a0b27f: Running coprocessor pre-open hook at 1733206921191Writing region info on filesystem at 1733206921191Initializing all the Stores at 1733206921193 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206921193Cleaning up temporary data from old regions at 1733206921207 (+14 ms)Running coprocessor post-open hooks at 1733206921217 (+10 ms)Region opened successfully at 1733206921217 2024-12-03T06:22:01,218 INFO [StoreOpener-6b148e6fa0896eea84e6c38d5293e44a-1 {}] regionserver.HStore(327): Store=6b148e6fa0896eea84e6c38d5293e44a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:01,218 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 6b148e6fa0896eea84e6c38d5293e44a 2024-12-03T06:22:01,219 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/6b148e6fa0896eea84e6c38d5293e44a 2024-12-03T06:22:01,220 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/6b148e6fa0896eea84e6c38d5293e44a 2024-12-03T06:22:01,221 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1733206919969.ae2330e378b6ad4b1a3e345529a0b27f., pid=49, masterSystemTime=1733206921141 2024-12-03T06:22:01,221 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure afa4ab371a599cc21eb81082233626e3, server=122cddc95ec8,45577,1733206917466 because future has completed 2024-12-03T06:22:01,221 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 6b148e6fa0896eea84e6c38d5293e44a 2024-12-03T06:22:01,222 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 6b148e6fa0896eea84e6c38d5293e44a 2024-12-03T06:22:01,223 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1733206919969.1e061778c0efa365f63836d0691ff7a8. 2024-12-03T06:22:01,223 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,ttt,1733206919969.1e061778c0efa365f63836d0691ff7a8. 2024-12-03T06:22:01,224 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,jjj,1733206919969.3c9e860d892b47160f56d24cb24e6010. 2024-12-03T06:22:01,224 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1733206919969.ae2330e378b6ad4b1a3e345529a0b27f. 2024-12-03T06:22:01,224 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,yyy,1733206919969.ae2330e378b6ad4b1a3e345529a0b27f. 2024-12-03T06:22:01,225 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,vvv,1733206919969.923ece5c25680df02ff5f922d838bc62. 2024-12-03T06:22:01,225 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 923ece5c25680df02ff5f922d838bc62, NAME => 'test,vvv,1733206919969.923ece5c25680df02ff5f922d838bc62.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-12-03T06:22:01,225 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 923ece5c25680df02ff5f922d838bc62 2024-12-03T06:22:01,225 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,vvv,1733206919969.923ece5c25680df02ff5f922d838bc62.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:01,225 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 923ece5c25680df02ff5f922d838bc62 2024-12-03T06:22:01,225 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 923ece5c25680df02ff5f922d838bc62 2024-12-03T06:22:01,226 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 3c9e860d892b47160f56d24cb24e6010, NAME => 'test,jjj,1733206919969.3c9e860d892b47160f56d24cb24e6010.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-12-03T06:22:01,226 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 6b148e6fa0896eea84e6c38d5293e44a 2024-12-03T06:22:01,227 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3c9e860d892b47160f56d24cb24e6010 2024-12-03T06:22:01,227 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,jjj,1733206919969.3c9e860d892b47160f56d24cb24e6010.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:01,227 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 3c9e860d892b47160f56d24cb24e6010 2024-12-03T06:22:01,227 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 3c9e860d892b47160f56d24cb24e6010 2024-12-03T06:22:01,228 INFO [StoreOpener-923ece5c25680df02ff5f922d838bc62-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 923ece5c25680df02ff5f922d838bc62 2024-12-03T06:22:01,229 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=23 2024-12-03T06:22:01,229 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 4e8636a0790f504025df15b4adf65735, server=122cddc95ec8,37369,1733206917509 in 221 msec 2024-12-03T06:22:01,230 INFO [StoreOpener-3c9e860d892b47160f56d24cb24e6010-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3c9e860d892b47160f56d24cb24e6010 2024-12-03T06:22:01,232 INFO [StoreOpener-923ece5c25680df02ff5f922d838bc62-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 923ece5c25680df02ff5f922d838bc62 columnFamilyName family 2024-12-03T06:22:01,232 DEBUG [StoreOpener-923ece5c25680df02ff5f922d838bc62-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:01,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN in 523 msec 2024-12-03T06:22:01,232 INFO [StoreOpener-3c9e860d892b47160f56d24cb24e6010-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c9e860d892b47160f56d24cb24e6010 columnFamilyName family 2024-12-03T06:22:01,232 INFO [StoreOpener-923ece5c25680df02ff5f922d838bc62-1 {}] regionserver.HStore(327): Store=923ece5c25680df02ff5f922d838bc62/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:01,233 DEBUG [StoreOpener-3c9e860d892b47160f56d24cb24e6010-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:01,233 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 923ece5c25680df02ff5f922d838bc62 2024-12-03T06:22:01,234 INFO [StoreOpener-3c9e860d892b47160f56d24cb24e6010-1 {}] regionserver.HStore(327): Store=3c9e860d892b47160f56d24cb24e6010/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:01,234 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/923ece5c25680df02ff5f922d838bc62 2024-12-03T06:22:01,234 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 3c9e860d892b47160f56d24cb24e6010 2024-12-03T06:22:01,234 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=ae2330e378b6ad4b1a3e345529a0b27f, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,37369,1733206917509 2024-12-03T06:22:01,234 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/923ece5c25680df02ff5f922d838bc62 2024-12-03T06:22:01,235 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/3c9e860d892b47160f56d24cb24e6010 2024-12-03T06:22:01,235 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 923ece5c25680df02ff5f922d838bc62 2024-12-03T06:22:01,235 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 923ece5c25680df02ff5f922d838bc62 2024-12-03T06:22:01,236 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/3c9e860d892b47160f56d24cb24e6010 2024-12-03T06:22:01,237 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 3c9e860d892b47160f56d24cb24e6010 2024-12-03T06:22:01,237 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 3c9e860d892b47160f56d24cb24e6010 2024-12-03T06:22:01,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae2330e378b6ad4b1a3e345529a0b27f, server=122cddc95ec8,37369,1733206917509 because future has completed 2024-12-03T06:22:01,240 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/6b148e6fa0896eea84e6c38d5293e44a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:22:01,241 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 6b148e6fa0896eea84e6c38d5293e44a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65565809, jitterRate=-0.022993311285972595}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:01,241 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6b148e6fa0896eea84e6c38d5293e44a 2024-12-03T06:22:01,242 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 923ece5c25680df02ff5f922d838bc62 2024-12-03T06:22:01,242 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 6b148e6fa0896eea84e6c38d5293e44a: Running coprocessor pre-open hook at 1733206921212Writing region info on filesystem at 1733206921212Initializing all the Stores at 1733206921213 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206921213Cleaning up temporary data from old regions at 1733206921222 (+9 ms)Running coprocessor post-open hooks at 1733206921242 (+20 ms)Region opened successfully at 1733206921242 2024-12-03T06:22:01,243 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1733206919969.6b148e6fa0896eea84e6c38d5293e44a., pid=31, masterSystemTime=1733206921031 2024-12-03T06:22:01,243 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 3c9e860d892b47160f56d24cb24e6010 2024-12-03T06:22:01,243 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=122cddc95ec8,45577,1733206917466, table=test, region=e3e1e4a86d09a1acb1fcf9d92f129898. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-03T06:22:01,245 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/923ece5c25680df02ff5f922d838bc62/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:22:01,246 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 923ece5c25680df02ff5f922d838bc62; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61246097, jitterRate=-0.08736203610897064}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:01,246 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/3c9e860d892b47160f56d24cb24e6010/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:22:01,247 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 923ece5c25680df02ff5f922d838bc62 2024-12-03T06:22:01,247 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 923ece5c25680df02ff5f922d838bc62: Running coprocessor pre-open hook at 1733206921226Writing region info on filesystem at 1733206921226Initializing all the Stores at 1733206921228 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206921228Cleaning up temporary data from old regions at 1733206921235 (+7 ms)Running coprocessor post-open hooks at 1733206921247 (+12 ms)Region opened successfully at 1733206921247 2024-12-03T06:22:01,247 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 3c9e860d892b47160f56d24cb24e6010; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68774747, jitterRate=0.024823591113090515}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:01,247 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3c9e860d892b47160f56d24cb24e6010 2024-12-03T06:22:01,248 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 3c9e860d892b47160f56d24cb24e6010: Running coprocessor pre-open hook at 1733206921227Writing region info on filesystem at 1733206921227Initializing all the Stores at 1733206921229 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206921229Cleaning up temporary data from old regions at 1733206921237 (+8 ms)Running coprocessor post-open hooks at 1733206921248 (+11 ms)Region opened successfully at 1733206921248 2024-12-03T06:22:01,249 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1733206919969.923ece5c25680df02ff5f922d838bc62., pid=53, masterSystemTime=1733206921141 2024-12-03T06:22:01,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=22 2024-12-03T06:22:01,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure afa4ab371a599cc21eb81082233626e3, server=122cddc95ec8,45577,1733206917466 in 306 msec 2024-12-03T06:22:01,249 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1733206919969.6b148e6fa0896eea84e6c38d5293e44a. 2024-12-03T06:22:01,249 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,xxx,1733206919969.6b148e6fa0896eea84e6c38d5293e44a. 2024-12-03T06:22:01,249 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1733206919969.3c9e860d892b47160f56d24cb24e6010., pid=43, masterSystemTime=1733206921055 2024-12-03T06:22:01,249 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,eee,1733206919969.e884340ece2e3eb17a321ed6d089e5e7. 2024-12-03T06:22:01,250 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=122cddc95ec8,37369,1733206917509, table=test, region=ae2330e378b6ad4b1a3e345529a0b27f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-03T06:22:01,250 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => e884340ece2e3eb17a321ed6d089e5e7, NAME => 'test,eee,1733206919969.e884340ece2e3eb17a321ed6d089e5e7.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-12-03T06:22:01,250 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e884340ece2e3eb17a321ed6d089e5e7 2024-12-03T06:22:01,250 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,eee,1733206919969.e884340ece2e3eb17a321ed6d089e5e7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:01,250 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for e884340ece2e3eb17a321ed6d089e5e7 2024-12-03T06:22:01,250 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for e884340ece2e3eb17a321ed6d089e5e7 2024-12-03T06:22:01,253 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1733206919969.3c9e860d892b47160f56d24cb24e6010. 2024-12-03T06:22:01,253 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,jjj,1733206919969.3c9e860d892b47160f56d24cb24e6010. 2024-12-03T06:22:01,253 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,www,1733206919969.e9f6e863b0ed7a188cb723f7a298ba2c. 2024-12-03T06:22:01,254 INFO [StoreOpener-e884340ece2e3eb17a321ed6d089e5e7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e884340ece2e3eb17a321ed6d089e5e7 2024-12-03T06:22:01,254 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => e9f6e863b0ed7a188cb723f7a298ba2c, NAME => 'test,www,1733206919969.e9f6e863b0ed7a188cb723f7a298ba2c.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-12-03T06:22:01,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=afa4ab371a599cc21eb81082233626e3, ASSIGN in 543 msec 2024-12-03T06:22:01,254 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e9f6e863b0ed7a188cb723f7a298ba2c 2024-12-03T06:22:01,254 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,www,1733206919969.e9f6e863b0ed7a188cb723f7a298ba2c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:01,254 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for e9f6e863b0ed7a188cb723f7a298ba2c 2024-12-03T06:22:01,254 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for e9f6e863b0ed7a188cb723f7a298ba2c 2024-12-03T06:22:01,256 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=3c9e860d892b47160f56d24cb24e6010, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:01,257 INFO [StoreOpener-e9f6e863b0ed7a188cb723f7a298ba2c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e9f6e863b0ed7a188cb723f7a298ba2c 2024-12-03T06:22:01,257 INFO [StoreOpener-e884340ece2e3eb17a321ed6d089e5e7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e884340ece2e3eb17a321ed6d089e5e7 columnFamilyName family 2024-12-03T06:22:01,258 DEBUG [StoreOpener-e884340ece2e3eb17a321ed6d089e5e7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:01,258 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1733206919969.923ece5c25680df02ff5f922d838bc62. 2024-12-03T06:22:01,258 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,vvv,1733206919969.923ece5c25680df02ff5f922d838bc62. 2024-12-03T06:22:01,258 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,fff,1733206919969.e48f9dd0d4bf7a7f5e5fb4dac950f301. 2024-12-03T06:22:01,258 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => e48f9dd0d4bf7a7f5e5fb4dac950f301, NAME => 'test,fff,1733206919969.e48f9dd0d4bf7a7f5e5fb4dac950f301.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-12-03T06:22:01,259 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e48f9dd0d4bf7a7f5e5fb4dac950f301 2024-12-03T06:22:01,259 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,fff,1733206919969.e48f9dd0d4bf7a7f5e5fb4dac950f301.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:01,259 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for e48f9dd0d4bf7a7f5e5fb4dac950f301 2024-12-03T06:22:01,259 INFO [StoreOpener-e884340ece2e3eb17a321ed6d089e5e7-1 {}] regionserver.HStore(327): Store=e884340ece2e3eb17a321ed6d089e5e7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:01,259 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for e48f9dd0d4bf7a7f5e5fb4dac950f301 2024-12-03T06:22:01,259 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for e884340ece2e3eb17a321ed6d089e5e7 2024-12-03T06:22:01,260 INFO [StoreOpener-e9f6e863b0ed7a188cb723f7a298ba2c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9f6e863b0ed7a188cb723f7a298ba2c columnFamilyName family 2024-12-03T06:22:01,260 DEBUG [StoreOpener-e9f6e863b0ed7a188cb723f7a298ba2c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:01,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c9e860d892b47160f56d24cb24e6010, server=122cddc95ec8,43629,1733206917367 because future has completed 2024-12-03T06:22:01,260 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/e884340ece2e3eb17a321ed6d089e5e7 2024-12-03T06:22:01,261 INFO [StoreOpener-e9f6e863b0ed7a188cb723f7a298ba2c-1 {}] regionserver.HStore(327): Store=e9f6e863b0ed7a188cb723f7a298ba2c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:01,261 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/e884340ece2e3eb17a321ed6d089e5e7 2024-12-03T06:22:01,261 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for e9f6e863b0ed7a188cb723f7a298ba2c 2024-12-03T06:22:01,261 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for e884340ece2e3eb17a321ed6d089e5e7 2024-12-03T06:22:01,261 INFO [StoreOpener-e48f9dd0d4bf7a7f5e5fb4dac950f301-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e48f9dd0d4bf7a7f5e5fb4dac950f301 2024-12-03T06:22:01,261 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for e884340ece2e3eb17a321ed6d089e5e7 2024-12-03T06:22:01,262 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/e9f6e863b0ed7a188cb723f7a298ba2c 2024-12-03T06:22:01,263 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/e9f6e863b0ed7a188cb723f7a298ba2c 2024-12-03T06:22:01,264 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for e9f6e863b0ed7a188cb723f7a298ba2c 2024-12-03T06:22:01,264 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for e9f6e863b0ed7a188cb723f7a298ba2c 2024-12-03T06:22:01,264 INFO [StoreOpener-e48f9dd0d4bf7a7f5e5fb4dac950f301-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e48f9dd0d4bf7a7f5e5fb4dac950f301 columnFamilyName family 2024-12-03T06:22:01,264 DEBUG [StoreOpener-e48f9dd0d4bf7a7f5e5fb4dac950f301-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:01,264 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for e884340ece2e3eb17a321ed6d089e5e7 2024-12-03T06:22:01,265 INFO [StoreOpener-e48f9dd0d4bf7a7f5e5fb4dac950f301-1 {}] regionserver.HStore(327): Store=e48f9dd0d4bf7a7f5e5fb4dac950f301/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:01,265 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for e48f9dd0d4bf7a7f5e5fb4dac950f301 2024-12-03T06:22:01,266 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/e48f9dd0d4bf7a7f5e5fb4dac950f301 2024-12-03T06:22:01,267 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=14 2024-12-03T06:22:01,267 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 3c9e860d892b47160f56d24cb24e6010, server=122cddc95ec8,43629,1733206917367 in 304 msec 2024-12-03T06:22:01,268 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/e48f9dd0d4bf7a7f5e5fb4dac950f301 2024-12-03T06:22:01,268 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for e9f6e863b0ed7a188cb723f7a298ba2c 2024-12-03T06:22:01,268 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for e48f9dd0d4bf7a7f5e5fb4dac950f301 2024-12-03T06:22:01,269 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for e48f9dd0d4bf7a7f5e5fb4dac950f301 2024-12-03T06:22:01,269 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/e884340ece2e3eb17a321ed6d089e5e7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:22:01,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3c9e860d892b47160f56d24cb24e6010, ASSIGN in 561 msec 2024-12-03T06:22:01,270 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened e884340ece2e3eb17a321ed6d089e5e7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61073620, jitterRate=-0.0899321436882019}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:01,272 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e884340ece2e3eb17a321ed6d089e5e7 2024-12-03T06:22:01,272 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for e884340ece2e3eb17a321ed6d089e5e7: Running coprocessor pre-open hook at 1733206921251Writing region info on filesystem at 1733206921251Initializing all the Stores at 1733206921252 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206921252Cleaning up temporary data from old regions at 1733206921261 (+9 ms)Running coprocessor post-open hooks at 1733206921272 (+11 ms)Region opened successfully at 1733206921272 2024-12-03T06:22:01,272 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for e48f9dd0d4bf7a7f5e5fb4dac950f301 2024-12-03T06:22:01,273 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1733206919969.e884340ece2e3eb17a321ed6d089e5e7., pid=36, masterSystemTime=1733206921031 2024-12-03T06:22:01,273 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/e9f6e863b0ed7a188cb723f7a298ba2c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:22:01,273 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=923ece5c25680df02ff5f922d838bc62, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,37369,1733206917509 2024-12-03T06:22:01,276 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened e9f6e863b0ed7a188cb723f7a298ba2c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66598302, jitterRate=-0.007607966661453247}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:01,276 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e9f6e863b0ed7a188cb723f7a298ba2c 2024-12-03T06:22:01,276 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for e9f6e863b0ed7a188cb723f7a298ba2c: Running coprocessor pre-open hook at 1733206921255Writing region info on filesystem at 1733206921255Initializing all the Stores at 1733206921256 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206921256Cleaning up temporary data from old regions at 1733206921264 (+8 ms)Running coprocessor post-open hooks at 1733206921276 (+12 ms)Region opened successfully at 1733206921276 2024-12-03T06:22:01,277 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 923ece5c25680df02ff5f922d838bc62, server=122cddc95ec8,37369,1733206917509 because future has completed 2024-12-03T06:22:01,277 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1733206919969.e884340ece2e3eb17a321ed6d089e5e7. 2024-12-03T06:22:01,278 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,eee,1733206919969.e884340ece2e3eb17a321ed6d089e5e7. 2024-12-03T06:22:01,278 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1733206919969.e9f6e863b0ed7a188cb723f7a298ba2c., pid=34, masterSystemTime=1733206921055 2024-12-03T06:22:01,278 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/e48f9dd0d4bf7a7f5e5fb4dac950f301/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:22:01,278 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,hhh,1733206919969.7d719355720f98b8bc93f39a6130075b. 2024-12-03T06:22:01,278 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 7d719355720f98b8bc93f39a6130075b, NAME => 'test,hhh,1733206919969.7d719355720f98b8bc93f39a6130075b.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-12-03T06:22:01,279 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7d719355720f98b8bc93f39a6130075b 2024-12-03T06:22:01,279 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,hhh,1733206919969.7d719355720f98b8bc93f39a6130075b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:01,279 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 7d719355720f98b8bc93f39a6130075b 2024-12-03T06:22:01,279 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 7d719355720f98b8bc93f39a6130075b 2024-12-03T06:22:01,279 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened e48f9dd0d4bf7a7f5e5fb4dac950f301; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72457759, jitterRate=0.07970474660396576}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:01,279 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e48f9dd0d4bf7a7f5e5fb4dac950f301 2024-12-03T06:22:01,279 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=6b148e6fa0896eea84e6c38d5293e44a, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:01,279 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for e48f9dd0d4bf7a7f5e5fb4dac950f301: Running coprocessor pre-open hook at 1733206921259Writing region info on filesystem at 1733206921259Initializing all the Stores at 1733206921260 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206921260Cleaning up temporary data from old regions at 1733206921269 (+9 ms)Running coprocessor post-open hooks at 1733206921279 (+10 ms)Region opened successfully at 1733206921279 2024-12-03T06:22:01,281 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1733206919969.e48f9dd0d4bf7a7f5e5fb4dac950f301., pid=56, masterSystemTime=1733206921141 2024-12-03T06:22:01,284 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1733206919969.e9f6e863b0ed7a188cb723f7a298ba2c. 2024-12-03T06:22:01,284 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,www,1733206919969.e9f6e863b0ed7a188cb723f7a298ba2c. 2024-12-03T06:22:01,284 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,nnn,1733206919969.b5f7ab205e6c897503e3ec729aaa3170. 2024-12-03T06:22:01,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6b148e6fa0896eea84e6c38d5293e44a, server=122cddc95ec8,45577,1733206917466 because future has completed 2024-12-03T06:22:01,284 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1733206919969.e48f9dd0d4bf7a7f5e5fb4dac950f301. 2024-12-03T06:22:01,285 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,fff,1733206919969.e48f9dd0d4bf7a7f5e5fb4dac950f301. 2024-12-03T06:22:01,285 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => b5f7ab205e6c897503e3ec729aaa3170, NAME => 'test,nnn,1733206919969.b5f7ab205e6c897503e3ec729aaa3170.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-12-03T06:22:01,285 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,lll,1733206919969.1ddf2a494d8d0fcd31eac685512f5644. 2024-12-03T06:22:01,285 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 1ddf2a494d8d0fcd31eac685512f5644, NAME => 'test,lll,1733206919969.1ddf2a494d8d0fcd31eac685512f5644.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-12-03T06:22:01,285 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b5f7ab205e6c897503e3ec729aaa3170 2024-12-03T06:22:01,285 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,nnn,1733206919969.b5f7ab205e6c897503e3ec729aaa3170.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:01,285 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=e9f6e863b0ed7a188cb723f7a298ba2c, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:01,285 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for b5f7ab205e6c897503e3ec729aaa3170 2024-12-03T06:22:01,285 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1ddf2a494d8d0fcd31eac685512f5644 2024-12-03T06:22:01,285 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for b5f7ab205e6c897503e3ec729aaa3170 2024-12-03T06:22:01,285 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,lll,1733206919969.1ddf2a494d8d0fcd31eac685512f5644.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:01,285 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 1ddf2a494d8d0fcd31eac685512f5644 2024-12-03T06:22:01,286 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 1ddf2a494d8d0fcd31eac685512f5644 2024-12-03T06:22:01,282 INFO [StoreOpener-7d719355720f98b8bc93f39a6130075b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7d719355720f98b8bc93f39a6130075b 2024-12-03T06:22:01,289 INFO [StoreOpener-b5f7ab205e6c897503e3ec729aaa3170-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b5f7ab205e6c897503e3ec729aaa3170 2024-12-03T06:22:01,289 INFO [StoreOpener-1ddf2a494d8d0fcd31eac685512f5644-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1ddf2a494d8d0fcd31eac685512f5644 2024-12-03T06:22:01,289 INFO [StoreOpener-7d719355720f98b8bc93f39a6130075b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7d719355720f98b8bc93f39a6130075b columnFamilyName family 2024-12-03T06:22:01,290 DEBUG [StoreOpener-7d719355720f98b8bc93f39a6130075b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:01,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure e9f6e863b0ed7a188cb723f7a298ba2c, server=122cddc95ec8,43629,1733206917367 because future has completed 2024-12-03T06:22:01,291 INFO [StoreOpener-7d719355720f98b8bc93f39a6130075b-1 {}] regionserver.HStore(327): Store=7d719355720f98b8bc93f39a6130075b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:01,292 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 7d719355720f98b8bc93f39a6130075b 2024-12-03T06:22:01,292 INFO [StoreOpener-b5f7ab205e6c897503e3ec729aaa3170-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5f7ab205e6c897503e3ec729aaa3170 columnFamilyName family 2024-12-03T06:22:01,292 DEBUG [StoreOpener-b5f7ab205e6c897503e3ec729aaa3170-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:01,292 INFO [StoreOpener-1ddf2a494d8d0fcd31eac685512f5644-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ddf2a494d8d0fcd31eac685512f5644 columnFamilyName family 2024-12-03T06:22:01,293 DEBUG [StoreOpener-1ddf2a494d8d0fcd31eac685512f5644-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:01,293 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/7d719355720f98b8bc93f39a6130075b 2024-12-03T06:22:01,293 INFO [StoreOpener-b5f7ab205e6c897503e3ec729aaa3170-1 {}] regionserver.HStore(327): Store=b5f7ab205e6c897503e3ec729aaa3170/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:01,293 INFO [StoreOpener-1ddf2a494d8d0fcd31eac685512f5644-1 {}] regionserver.HStore(327): Store=1ddf2a494d8d0fcd31eac685512f5644/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:01,294 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for b5f7ab205e6c897503e3ec729aaa3170 2024-12-03T06:22:01,294 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/7d719355720f98b8bc93f39a6130075b 2024-12-03T06:22:01,294 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 1ddf2a494d8d0fcd31eac685512f5644 2024-12-03T06:22:01,295 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 7d719355720f98b8bc93f39a6130075b 2024-12-03T06:22:01,295 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 7d719355720f98b8bc93f39a6130075b 2024-12-03T06:22:01,299 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/1ddf2a494d8d0fcd31eac685512f5644 2024-12-03T06:22:01,299 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/b5f7ab205e6c897503e3ec729aaa3170 2024-12-03T06:22:01,300 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/1ddf2a494d8d0fcd31eac685512f5644 2024-12-03T06:22:01,300 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/b5f7ab205e6c897503e3ec729aaa3170 2024-12-03T06:22:01,300 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for b5f7ab205e6c897503e3ec729aaa3170 2024-12-03T06:22:01,301 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 1ddf2a494d8d0fcd31eac685512f5644 2024-12-03T06:22:01,301 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for b5f7ab205e6c897503e3ec729aaa3170 2024-12-03T06:22:01,301 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 1ddf2a494d8d0fcd31eac685512f5644 2024-12-03T06:22:01,301 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=28 2024-12-03T06:22:01,301 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 6b148e6fa0896eea84e6c38d5293e44a, server=122cddc95ec8,45577,1733206917466 in 415 msec 2024-12-03T06:22:01,303 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 7d719355720f98b8bc93f39a6130075b 2024-12-03T06:22:01,304 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for b5f7ab205e6c897503e3ec729aaa3170 2024-12-03T06:22:01,304 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 1ddf2a494d8d0fcd31eac685512f5644 2024-12-03T06:22:01,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6b148e6fa0896eea84e6c38d5293e44a, ASSIGN in 595 msec 2024-12-03T06:22:01,311 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/b5f7ab205e6c897503e3ec729aaa3170/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:22:01,311 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/7d719355720f98b8bc93f39a6130075b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:22:01,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=27 2024-12-03T06:22:01,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure e9f6e863b0ed7a188cb723f7a298ba2c, server=122cddc95ec8,43629,1733206917367 in 405 msec 2024-12-03T06:22:01,312 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 7d719355720f98b8bc93f39a6130075b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71528731, jitterRate=0.06586115062236786}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:01,312 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened b5f7ab205e6c897503e3ec729aaa3170; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75355325, jitterRate=0.12288184463977814}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:01,312 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7d719355720f98b8bc93f39a6130075b 2024-12-03T06:22:01,312 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b5f7ab205e6c897503e3ec729aaa3170 2024-12-03T06:22:01,312 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for b5f7ab205e6c897503e3ec729aaa3170: Running coprocessor pre-open hook at 1733206921285Writing region info on filesystem at 1733206921285Initializing all the Stores at 1733206921288 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206921288Cleaning up temporary data from old regions at 1733206921301 (+13 ms)Running coprocessor post-open hooks at 1733206921312 (+11 ms)Region opened successfully at 1733206921312 2024-12-03T06:22:01,312 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 7d719355720f98b8bc93f39a6130075b: Running coprocessor pre-open hook at 1733206921279Writing region info on filesystem at 1733206921279Initializing all the Stores at 1733206921281 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206921281Cleaning up temporary data from old regions at 1733206921295 (+14 ms)Running coprocessor post-open hooks at 1733206921312 (+17 ms)Region opened successfully at 1733206921312 2024-12-03T06:22:01,313 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1733206919969.b5f7ab205e6c897503e3ec729aaa3170., pid=47, masterSystemTime=1733206921055 2024-12-03T06:22:01,314 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1733206919969.7d719355720f98b8bc93f39a6130075b., pid=37, masterSystemTime=1733206921031 2024-12-03T06:22:01,315 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/1ddf2a494d8d0fcd31eac685512f5644/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:22:01,316 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 1ddf2a494d8d0fcd31eac685512f5644; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71141626, jitterRate=0.06009283661842346}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:01,316 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1ddf2a494d8d0fcd31eac685512f5644 2024-12-03T06:22:01,316 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 1ddf2a494d8d0fcd31eac685512f5644: Running coprocessor pre-open hook at 1733206921286Writing region info on filesystem at 1733206921286Initializing all the Stores at 1733206921288 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206921288Cleaning up temporary data from old regions at 1733206921301 (+13 ms)Running coprocessor post-open hooks at 1733206921316 (+15 ms)Region opened successfully at 1733206921316 2024-12-03T06:22:01,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e9f6e863b0ed7a188cb723f7a298ba2c, ASSIGN in 606 msec 2024-12-03T06:22:01,318 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1733206919969.7d719355720f98b8bc93f39a6130075b. 2024-12-03T06:22:01,318 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,hhh,1733206919969.7d719355720f98b8bc93f39a6130075b. 2024-12-03T06:22:01,318 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1733206919969.1ddf2a494d8d0fcd31eac685512f5644., pid=50, masterSystemTime=1733206921141 2024-12-03T06:22:01,319 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=7d719355720f98b8bc93f39a6130075b, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:01,319 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1733206919969.b5f7ab205e6c897503e3ec729aaa3170. 2024-12-03T06:22:01,319 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,nnn,1733206919969.b5f7ab205e6c897503e3ec729aaa3170. 2024-12-03T06:22:01,319 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,ddd,1733206919969.33124cac09cf3a2502ce5e473bf614f4. 2024-12-03T06:22:01,319 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 33124cac09cf3a2502ce5e473bf614f4, NAME => 'test,ddd,1733206919969.33124cac09cf3a2502ce5e473bf614f4.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-12-03T06:22:01,320 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 33124cac09cf3a2502ce5e473bf614f4 2024-12-03T06:22:01,320 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,ddd,1733206919969.33124cac09cf3a2502ce5e473bf614f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:01,320 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 33124cac09cf3a2502ce5e473bf614f4 2024-12-03T06:22:01,320 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 33124cac09cf3a2502ce5e473bf614f4 2024-12-03T06:22:01,322 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1733206919969.1ddf2a494d8d0fcd31eac685512f5644. 2024-12-03T06:22:01,322 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,lll,1733206919969.1ddf2a494d8d0fcd31eac685512f5644. 2024-12-03T06:22:01,322 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,iii,1733206919969.a332f3da50b1f1fe35aa4e0307fe35f8. 2024-12-03T06:22:01,322 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => a332f3da50b1f1fe35aa4e0307fe35f8, NAME => 'test,iii,1733206919969.a332f3da50b1f1fe35aa4e0307fe35f8.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-12-03T06:22:01,323 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a332f3da50b1f1fe35aa4e0307fe35f8 2024-12-03T06:22:01,323 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,iii,1733206919969.a332f3da50b1f1fe35aa4e0307fe35f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:01,323 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for a332f3da50b1f1fe35aa4e0307fe35f8 2024-12-03T06:22:01,323 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for a332f3da50b1f1fe35aa4e0307fe35f8 2024-12-03T06:22:01,324 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7d719355720f98b8bc93f39a6130075b, server=122cddc95ec8,45577,1733206917466 because future has completed 2024-12-03T06:22:01,325 INFO [StoreOpener-33124cac09cf3a2502ce5e473bf614f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 33124cac09cf3a2502ce5e473bf614f4 2024-12-03T06:22:01,325 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1ddf2a494d8d0fcd31eac685512f5644, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,37369,1733206917509 2024-12-03T06:22:01,326 INFO [StoreOpener-a332f3da50b1f1fe35aa4e0307fe35f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a332f3da50b1f1fe35aa4e0307fe35f8 2024-12-03T06:22:01,327 INFO [StoreOpener-33124cac09cf3a2502ce5e473bf614f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33124cac09cf3a2502ce5e473bf614f4 columnFamilyName family 2024-12-03T06:22:01,327 DEBUG [StoreOpener-33124cac09cf3a2502ce5e473bf614f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:01,328 INFO [StoreOpener-33124cac09cf3a2502ce5e473bf614f4-1 {}] regionserver.HStore(327): Store=33124cac09cf3a2502ce5e473bf614f4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:01,329 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ddf2a494d8d0fcd31eac685512f5644, server=122cddc95ec8,37369,1733206917509 because future has completed 2024-12-03T06:22:01,329 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 33124cac09cf3a2502ce5e473bf614f4 2024-12-03T06:22:01,330 INFO [StoreOpener-a332f3da50b1f1fe35aa4e0307fe35f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a332f3da50b1f1fe35aa4e0307fe35f8 columnFamilyName family 2024-12-03T06:22:01,330 DEBUG [StoreOpener-a332f3da50b1f1fe35aa4e0307fe35f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:01,330 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/33124cac09cf3a2502ce5e473bf614f4 2024-12-03T06:22:01,331 INFO [StoreOpener-a332f3da50b1f1fe35aa4e0307fe35f8-1 {}] regionserver.HStore(327): Store=a332f3da50b1f1fe35aa4e0307fe35f8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:01,331 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for a332f3da50b1f1fe35aa4e0307fe35f8 2024-12-03T06:22:01,331 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/33124cac09cf3a2502ce5e473bf614f4 2024-12-03T06:22:01,333 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/a332f3da50b1f1fe35aa4e0307fe35f8 2024-12-03T06:22:01,333 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 33124cac09cf3a2502ce5e473bf614f4 2024-12-03T06:22:01,333 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 33124cac09cf3a2502ce5e473bf614f4 2024-12-03T06:22:01,333 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/a332f3da50b1f1fe35aa4e0307fe35f8 2024-12-03T06:22:01,334 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for a332f3da50b1f1fe35aa4e0307fe35f8 2024-12-03T06:22:01,334 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for a332f3da50b1f1fe35aa4e0307fe35f8 2024-12-03T06:22:01,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=12 2024-12-03T06:22:01,335 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 33124cac09cf3a2502ce5e473bf614f4 2024-12-03T06:22:01,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 7d719355720f98b8bc93f39a6130075b, server=122cddc95ec8,45577,1733206917466 in 415 msec 2024-12-03T06:22:01,336 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for a332f3da50b1f1fe35aa4e0307fe35f8 2024-12-03T06:22:01,338 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/33124cac09cf3a2502ce5e473bf614f4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:22:01,339 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 33124cac09cf3a2502ce5e473bf614f4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69465942, jitterRate=0.035123199224472046}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:01,340 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 33124cac09cf3a2502ce5e473bf614f4 2024-12-03T06:22:01,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7d719355720f98b8bc93f39a6130075b, ASSIGN in 629 msec 2024-12-03T06:22:01,340 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/a332f3da50b1f1fe35aa4e0307fe35f8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:22:01,340 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 33124cac09cf3a2502ce5e473bf614f4: Running coprocessor pre-open hook at 1733206921320Writing region info on filesystem at 1733206921320Initializing all the Stores at 1733206921324 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206921324Cleaning up temporary data from old regions at 1733206921333 (+9 ms)Running coprocessor post-open hooks at 1733206921340 (+7 ms)Region opened successfully at 1733206921340 2024-12-03T06:22:01,341 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened a332f3da50b1f1fe35aa4e0307fe35f8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65928884, jitterRate=-0.017583072185516357}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:01,341 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a332f3da50b1f1fe35aa4e0307fe35f8 2024-12-03T06:22:01,341 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for a332f3da50b1f1fe35aa4e0307fe35f8: Running coprocessor pre-open hook at 1733206921323Writing region info on filesystem at 1733206921323Initializing all the Stores at 1733206921325 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206921325Cleaning up temporary data from old regions at 1733206921334 (+9 ms)Running coprocessor post-open hooks at 1733206921341 (+7 ms)Region opened successfully at 1733206921341 2024-12-03T06:22:01,341 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1733206919969.33124cac09cf3a2502ce5e473bf614f4., pid=45, masterSystemTime=1733206921055 2024-12-03T06:22:01,342 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1733206919969.a332f3da50b1f1fe35aa4e0307fe35f8., pid=54, masterSystemTime=1733206921141 2024-12-03T06:22:01,348 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1733206919969.33124cac09cf3a2502ce5e473bf614f4. 2024-12-03T06:22:01,349 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,ddd,1733206919969.33124cac09cf3a2502ce5e473bf614f4. 2024-12-03T06:22:01,350 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1733206919969.a332f3da50b1f1fe35aa4e0307fe35f8. 2024-12-03T06:22:01,350 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=122cddc95ec8,45577,1733206917466, table=test, region=e3e1e4a86d09a1acb1fcf9d92f129898. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-03T06:22:01,350 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,iii,1733206919969.a332f3da50b1f1fe35aa4e0307fe35f8. 2024-12-03T06:22:01,351 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,ppp,1733206919969.07dbe58fce62a9e98b77016cec645f5e. 2024-12-03T06:22:01,351 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 07dbe58fce62a9e98b77016cec645f5e, NAME => 'test,ppp,1733206919969.07dbe58fce62a9e98b77016cec645f5e.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-12-03T06:22:01,351 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 07dbe58fce62a9e98b77016cec645f5e 2024-12-03T06:22:01,351 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,ppp,1733206919969.07dbe58fce62a9e98b77016cec645f5e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:01,351 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 07dbe58fce62a9e98b77016cec645f5e 2024-12-03T06:22:01,351 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 07dbe58fce62a9e98b77016cec645f5e 2024-12-03T06:22:01,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=16 2024-12-03T06:22:01,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1ddf2a494d8d0fcd31eac685512f5644, server=122cddc95ec8,37369,1733206917509 in 348 msec 2024-12-03T06:22:01,353 INFO [StoreOpener-07dbe58fce62a9e98b77016cec645f5e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 07dbe58fce62a9e98b77016cec645f5e 2024-12-03T06:22:01,356 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=122cddc95ec8,37369,1733206917509, table=test, region=923ece5c25680df02ff5f922d838bc62. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-03T06:22:01,356 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=122cddc95ec8,37369,1733206917509, table=test, region=ae2330e378b6ad4b1a3e345529a0b27f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-03T06:22:01,356 INFO [StoreOpener-07dbe58fce62a9e98b77016cec645f5e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07dbe58fce62a9e98b77016cec645f5e columnFamilyName family 2024-12-03T06:22:01,356 DEBUG [StoreOpener-07dbe58fce62a9e98b77016cec645f5e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:01,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN in 646 msec 2024-12-03T06:22:01,357 INFO [StoreOpener-07dbe58fce62a9e98b77016cec645f5e-1 {}] regionserver.HStore(327): Store=07dbe58fce62a9e98b77016cec645f5e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:01,357 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 07dbe58fce62a9e98b77016cec645f5e 2024-12-03T06:22:01,358 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/07dbe58fce62a9e98b77016cec645f5e 2024-12-03T06:22:01,358 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=a332f3da50b1f1fe35aa4e0307fe35f8, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,37369,1733206917509 2024-12-03T06:22:01,359 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/07dbe58fce62a9e98b77016cec645f5e 2024-12-03T06:22:01,359 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 07dbe58fce62a9e98b77016cec645f5e 2024-12-03T06:22:01,359 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 07dbe58fce62a9e98b77016cec645f5e 2024-12-03T06:22:01,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure a332f3da50b1f1fe35aa4e0307fe35f8, server=122cddc95ec8,37369,1733206917509 because future has completed 2024-12-03T06:22:01,362 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 07dbe58fce62a9e98b77016cec645f5e 2024-12-03T06:22:01,362 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=33124cac09cf3a2502ce5e473bf614f4, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:01,365 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 33124cac09cf3a2502ce5e473bf614f4, server=122cddc95ec8,43629,1733206917367 because future has completed 2024-12-03T06:22:01,366 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/07dbe58fce62a9e98b77016cec645f5e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:22:01,366 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 07dbe58fce62a9e98b77016cec645f5e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59778593, jitterRate=-0.10922954976558685}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:01,366 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 07dbe58fce62a9e98b77016cec645f5e 2024-12-03T06:22:01,367 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 07dbe58fce62a9e98b77016cec645f5e: Running coprocessor pre-open hook at 1733206921352Writing region info on filesystem at 1733206921352Initializing all the Stores at 1733206921353 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206921353Cleaning up temporary data from old regions at 1733206921359 (+6 ms)Running coprocessor post-open hooks at 1733206921366 (+7 ms)Region opened successfully at 1733206921367 (+1 ms) 2024-12-03T06:22:01,367 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1733206919969.07dbe58fce62a9e98b77016cec645f5e., pid=51, masterSystemTime=1733206921141 2024-12-03T06:22:01,370 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1733206919969.07dbe58fce62a9e98b77016cec645f5e. 2024-12-03T06:22:01,371 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,ppp,1733206919969.07dbe58fce62a9e98b77016cec645f5e. 2024-12-03T06:22:01,371 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,,1733206919969.663ef4d3dbbcf7b1bb25bba1055da980. 2024-12-03T06:22:01,371 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 663ef4d3dbbcf7b1bb25bba1055da980, NAME => 'test,,1733206919969.663ef4d3dbbcf7b1bb25bba1055da980.', STARTKEY => '', ENDKEY => 'bbb'} 2024-12-03T06:22:01,371 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 663ef4d3dbbcf7b1bb25bba1055da980 2024-12-03T06:22:01,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=8 2024-12-03T06:22:01,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 33124cac09cf3a2502ce5e473bf614f4, server=122cddc95ec8,43629,1733206917367 in 400 msec 2024-12-03T06:22:01,372 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,,1733206919969.663ef4d3dbbcf7b1bb25bba1055da980.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:01,372 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 663ef4d3dbbcf7b1bb25bba1055da980 2024-12-03T06:22:01,372 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 663ef4d3dbbcf7b1bb25bba1055da980 2024-12-03T06:22:01,374 INFO [StoreOpener-663ef4d3dbbcf7b1bb25bba1055da980-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 663ef4d3dbbcf7b1bb25bba1055da980 2024-12-03T06:22:01,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=33124cac09cf3a2502ce5e473bf614f4, ASSIGN in 666 msec 2024-12-03T06:22:01,376 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=07dbe58fce62a9e98b77016cec645f5e, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,37369,1733206917509 2024-12-03T06:22:01,377 INFO [StoreOpener-663ef4d3dbbcf7b1bb25bba1055da980-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 663ef4d3dbbcf7b1bb25bba1055da980 columnFamilyName family 2024-12-03T06:22:01,377 DEBUG [StoreOpener-663ef4d3dbbcf7b1bb25bba1055da980-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:01,379 INFO [StoreOpener-663ef4d3dbbcf7b1bb25bba1055da980-1 {}] regionserver.HStore(327): Store=663ef4d3dbbcf7b1bb25bba1055da980/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:01,379 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 663ef4d3dbbcf7b1bb25bba1055da980 2024-12-03T06:22:01,380 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/663ef4d3dbbcf7b1bb25bba1055da980 2024-12-03T06:22:01,380 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07dbe58fce62a9e98b77016cec645f5e, server=122cddc95ec8,37369,1733206917509 because future has completed 2024-12-03T06:22:01,381 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/663ef4d3dbbcf7b1bb25bba1055da980 2024-12-03T06:22:01,381 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 663ef4d3dbbcf7b1bb25bba1055da980 2024-12-03T06:22:01,381 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 663ef4d3dbbcf7b1bb25bba1055da980 2024-12-03T06:22:01,384 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 663ef4d3dbbcf7b1bb25bba1055da980 2024-12-03T06:22:01,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=13 2024-12-03T06:22:01,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure a332f3da50b1f1fe35aa4e0307fe35f8, server=122cddc95ec8,37369,1733206917509 in 368 msec 2024-12-03T06:22:01,387 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN in 679 msec 2024-12-03T06:22:01,388 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/663ef4d3dbbcf7b1bb25bba1055da980/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T06:22:01,389 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 663ef4d3dbbcf7b1bb25bba1055da980; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63003488, jitterRate=-0.061174869537353516}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:01,389 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 663ef4d3dbbcf7b1bb25bba1055da980 2024-12-03T06:22:01,389 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 663ef4d3dbbcf7b1bb25bba1055da980: Running coprocessor pre-open hook at 1733206921372Writing region info on filesystem at 1733206921372Initializing all the Stores at 1733206921374 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206921374Cleaning up temporary data from old regions at 1733206921381 (+7 ms)Running coprocessor post-open hooks at 1733206921389 (+8 ms)Region opened successfully at 1733206921389 2024-12-03T06:22:01,390 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1733206919969.663ef4d3dbbcf7b1bb25bba1055da980., pid=55, masterSystemTime=1733206921141 2024-12-03T06:22:01,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=20 2024-12-03T06:22:01,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 07dbe58fce62a9e98b77016cec645f5e, server=122cddc95ec8,37369,1733206917509 in 389 msec 2024-12-03T06:22:01,392 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1733206919969.663ef4d3dbbcf7b1bb25bba1055da980. 2024-12-03T06:22:01,392 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,,1733206919969.663ef4d3dbbcf7b1bb25bba1055da980. 2024-12-03T06:22:01,393 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=663ef4d3dbbcf7b1bb25bba1055da980, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,37369,1733206917509 2024-12-03T06:22:01,396 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 663ef4d3dbbcf7b1bb25bba1055da980, server=122cddc95ec8,37369,1733206917509 because future has completed 2024-12-03T06:22:01,398 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN in 690 msec 2024-12-03T06:22:01,401 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=5 2024-12-03T06:22:01,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 663ef4d3dbbcf7b1bb25bba1055da980, server=122cddc95ec8,37369,1733206917509 in 381 msec 2024-12-03T06:22:01,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN in 696 msec 2024-12-03T06:22:01,405 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=b5f7ab205e6c897503e3ec729aaa3170, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:01,408 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure b5f7ab205e6c897503e3ec729aaa3170, server=122cddc95ec8,43629,1733206917367 because future has completed 2024-12-03T06:22:01,409 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=e48f9dd0d4bf7a7f5e5fb4dac950f301, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,37369,1733206917509 2024-12-03T06:22:01,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,37369,1733206917509 because future has completed 2024-12-03T06:22:01,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=18 2024-12-03T06:22:01,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure b5f7ab205e6c897503e3ec729aaa3170, server=122cddc95ec8,43629,1733206917367 in 436 msec 2024-12-03T06:22:01,418 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b5f7ab205e6c897503e3ec729aaa3170, ASSIGN in 710 msec 2024-12-03T06:22:01,422 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=10 2024-12-03T06:22:01,422 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,37369,1733206917509 in 396 msec 2024-12-03T06:22:01,424 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN in 716 msec 2024-12-03T06:22:01,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=26 2024-12-03T06:22:01,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 923ece5c25680df02ff5f922d838bc62, server=122cddc95ec8,37369,1733206917509 in 419 msec 2024-12-03T06:22:01,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN in 724 msec 2024-12-03T06:22:01,434 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=e884340ece2e3eb17a321ed6d089e5e7, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:01,437 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure e884340ece2e3eb17a321ed6d089e5e7, server=122cddc95ec8,45577,1733206917466 because future has completed 2024-12-03T06:22:01,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=29 2024-12-03T06:22:01,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure ae2330e378b6ad4b1a3e345529a0b27f, server=122cddc95ec8,37369,1733206917509 in 450 msec 2024-12-03T06:22:01,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN in 735 msec 2024-12-03T06:22:01,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=9 2024-12-03T06:22:01,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure e884340ece2e3eb17a321ed6d089e5e7, server=122cddc95ec8,45577,1733206917466 in 534 msec 2024-12-03T06:22:01,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e884340ece2e3eb17a321ed6d089e5e7, ASSIGN in 743 msec 2024-12-03T06:22:01,455 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=122cddc95ec8,45577,1733206917466, table=test, region=e3e1e4a86d09a1acb1fcf9d92f129898. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-03T06:22:01,456 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=1e061778c0efa365f63836d0691ff7a8, regionState=OPEN, openSeqNum=2, regionLocation=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:01,460 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e061778c0efa365f63836d0691ff7a8, server=122cddc95ec8,43629,1733206917367 because future has completed 2024-12-03T06:22:01,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=25 2024-12-03T06:22:01,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure e3e1e4a86d09a1acb1fcf9d92f129898, server=122cddc95ec8,45577,1733206917466 in 521 msec 2024-12-03T06:22:01,467 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e3e1e4a86d09a1acb1fcf9d92f129898, ASSIGN in 759 msec 2024-12-03T06:22:01,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=24 2024-12-03T06:22:01,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 1e061778c0efa365f63836d0691ff7a8, server=122cddc95ec8,43629,1733206917367 in 517 msec 2024-12-03T06:22:01,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=4 2024-12-03T06:22:01,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1e061778c0efa365f63836d0691ff7a8, ASSIGN in 766 msec 2024-12-03T06:22:01,479 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T06:22:01,479 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733206921479"}]},"ts":"1733206921479"} 2024-12-03T06:22:01,482 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-12-03T06:22:01,484 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T06:22:01,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.5050 sec 2024-12-03T06:22:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T06:22:02,156 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-12-03T06:22:02,156 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-12-03T06:22:02,159 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T06:22:02,183 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-12-03T06:22:02,183 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T06:22:02,184 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-12-03T06:22:02,185 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T06:22:02,191 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-12-03T06:22:02,221 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-12-03T06:22:02,243 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:22:02,246 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:22:02,249 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:22:02,252 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:22:02,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 663ef4d3dbbcf7b1bb25bba1055da980 changed from -1.0 to 0.0, refreshing cache 2024-12-03T06:22:02,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e48f9dd0d4bf7a7f5e5fb4dac950f301 changed from -1.0 to 0.0, refreshing cache 2024-12-03T06:22:02,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a332f3da50b1f1fe35aa4e0307fe35f8 changed from -1.0 to 0.0, refreshing cache 2024-12-03T06:22:02,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1ddf2a494d8d0fcd31eac685512f5644 changed from -1.0 to 0.0, refreshing cache 2024-12-03T06:22:02,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 07dbe58fce62a9e98b77016cec645f5e changed from -1.0 to 0.0, refreshing cache 2024-12-03T06:22:02,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4e8636a0790f504025df15b4adf65735 changed from -1.0 to 0.0, refreshing cache 2024-12-03T06:22:02,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 923ece5c25680df02ff5f922d838bc62 changed from -1.0 to 0.0, refreshing cache 2024-12-03T06:22:02,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ae2330e378b6ad4b1a3e345529a0b27f changed from -1.0 to 0.0, refreshing cache 2024-12-03T06:22:02,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 40b4b0967914401ebc70c24d4b56a381 changed from -1.0 to 0.0, refreshing cache 2024-12-03T06:22:02,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 33124cac09cf3a2502ce5e473bf614f4 changed from -1.0 to 0.0, refreshing cache 2024-12-03T06:22:02,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ca916e60e459eeefe786a877b8700115 changed from -1.0 to 0.0, refreshing cache 2024-12-03T06:22:02,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3c9e860d892b47160f56d24cb24e6010 changed from -1.0 to 0.0, refreshing cache 2024-12-03T06:22:02,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b5f7ab205e6c897503e3ec729aaa3170 changed from -1.0 to 0.0, refreshing cache 2024-12-03T06:22:02,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6badf4f9ef3213a92a5f6ace694c6482 changed from -1.0 to 0.0, refreshing cache 2024-12-03T06:22:02,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1e061778c0efa365f63836d0691ff7a8 changed from -1.0 to 0.0, refreshing cache 2024-12-03T06:22:02,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e9f6e863b0ed7a188cb723f7a298ba2c changed from -1.0 to 0.0, refreshing cache 2024-12-03T06:22:02,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 671297fbd74543fd2b2deed06d08e6a7 changed from -1.0 to 0.0, refreshing cache 2024-12-03T06:22:02,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3bbc9d78d890c8d774a80646d9983ed6 changed from -1.0 to 0.0, refreshing cache 2024-12-03T06:22:02,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e884340ece2e3eb17a321ed6d089e5e7 changed from -1.0 to 0.0, refreshing cache 2024-12-03T06:22:02,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7d719355720f98b8bc93f39a6130075b changed from -1.0 to 0.0, refreshing cache 2024-12-03T06:22:02,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f495460f3e799a3e64378f8879c721bb changed from -1.0 to 0.0, refreshing cache 2024-12-03T06:22:02,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 050ab97be4cc9f90a155d3b261335a0d changed from -1.0 to 0.0, refreshing cache 2024-12-03T06:22:02,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a6fcc2e40df410122347a4873dfd1129 changed from -1.0 to 0.0, refreshing cache 2024-12-03T06:22:02,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region afa4ab371a599cc21eb81082233626e3 changed from -1.0 to 0.0, refreshing cache 2024-12-03T06:22:02,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e3e1e4a86d09a1acb1fcf9d92f129898 changed from -1.0 to 0.0, refreshing cache 2024-12-03T06:22:02,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6b148e6fa0896eea84e6c38d5293e44a changed from -1.0 to 0.0, refreshing cache 2024-12-03T06:22:02,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-12-03T06:22:02,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-12-03T06:22:02,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-12-03T06:22:02,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.BalancerClusterState(204): Hosts are {122cddc95ec8=0} racks are {/default-rack=0} 2024-12-03T06:22:02,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-12-03T06:22:02,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-12-03T06:22:02,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-12-03T06:22:02,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T06:22:02,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T06:22:02,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-03T06:22:02,323 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T06:22:02,323 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T06:22:02,324 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-03T06:22:02,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-12-03T06:22:02,329 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] balancer.StochasticLoadBalancer(395): Cluster wide - skipping load balancing because weighted average imbalance=0.016474464579901153 <= threshold(0.025). If you want more aggressive balancing, either lower hbase.master.balancer.stochastic.minCostNeedBalance from 0.025 or increase the relative multiplier(s) of the specific cost function(s). functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-12-03T06:22:02,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-12-03T06:22:02,331 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T06:22:02,341 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-12-03T06:22:02,361 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=380, OpenFileDescriptor=639, MaxFileDescriptor=1048576, SystemLoadAverage=301, ProcessCount=11, AvailableMemoryMB=12093 2024-12-03T06:22:02,365 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-12-03T06:22:02,366 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-12-03T06:22:02,366 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 122cddc95ec8,37369,1733206917509 2024-12-03T06:22:02,367 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 122cddc95ec8,37369,1733206917509: Simulated kill ***** 2024-12-03T06:22:02,367 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-03T06:22:02,369 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-03T06:22:02,371 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-03T06:22:02,372 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-03T06:22:02,372 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-03T06:22:02,372 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 304839896 }, "NonHeapMemoryUsage": { "committed": 151453696, "init": 7667712, "max": -1, "used": 149185776 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "122cddc95ec8" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "122cddc95ec8" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "122cddc95ec8" } ] } 2024-12-03T06:22:02,375 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35025 {}] master.MasterRpcServices(700): 122cddc95ec8,37369,1733206917509 reported a fatal error: ***** ABORTING region server 122cddc95ec8,37369,1733206917509: Simulated kill ***** 2024-12-03T06:22:02,377 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '122cddc95ec8,37369,1733206917509' ***** 2024-12-03T06:22:02,378 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-12-03T06:22:02,378 INFO [RS:2;122cddc95ec8:37369 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T06:22:02,378 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T06:22:02,378 INFO [RS:2;122cddc95ec8:37369 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-03T06:22:02,378 INFO [RS:2;122cddc95ec8:37369 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-03T06:22:02,378 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T06:22:02,379 INFO [RS:2;122cddc95ec8:37369 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T06:22:02,379 INFO [RS:2;122cddc95ec8:37369 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;122cddc95ec8:37369. 2024-12-03T06:22:02,379 DEBUG [RS:2;122cddc95ec8:37369 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T06:22:02,384 DEBUG [RS:2;122cddc95ec8:37369 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:22:02,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741836_1012 (size=32114) 2024-12-03T06:22:02,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741836_1012 (size=32114) 2024-12-03T06:22:02,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741836_1012 (size=32114) 2024-12-03T06:22:02,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741834_1010 (size=2232) 2024-12-03T06:22:02,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741834_1010 (size=2232) 2024-12-03T06:22:02,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741834_1010 (size=2232) 2024-12-03T06:22:02,395 DEBUG [RS:2;122cddc95ec8:37369 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:22:02,395 INFO [RS:2;122cddc95ec8:37369 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T06:22:02,395 INFO [RS:2;122cddc95ec8:37369 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37369 2024-12-03T06:22:02,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T06:22:02,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37369-0x1009eb0b9ca0003, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/122cddc95ec8,37369,1733206917509 2024-12-03T06:22:02,436 INFO [RS:2;122cddc95ec8:37369 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T06:22:02,436 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$365/0x00007fdfc48c4260@1fd7db3e rejected from java.util.concurrent.ThreadPoolExecutor@5ad556a0[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 12] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1360) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-12-03T06:22:02,444 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [122cddc95ec8,37369,1733206917509] 2024-12-03T06:22:02,451 INFO [regionserver/122cddc95ec8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T06:22:02,452 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/122cddc95ec8,37369,1733206917509 already deleted, retry=false 2024-12-03T06:22:02,452 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 122cddc95ec8,37369,1733206917509 on 122cddc95ec8,35025,1733206916647 2024-12-03T06:22:02,462 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true 2024-12-03T06:22:02,465 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=57 for 122cddc95ec8,37369,1733206917509 (carryingMeta=true) 122cddc95ec8,37369,1733206917509/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@2fa9ba6d[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-03T06:22:02,466 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true 2024-12-03T06:22:02,467 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true, isMeta: true 2024-12-03T06:22:02,469 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509-splitting 2024-12-03T06:22:02,471 INFO [PEWorker-1 {}] master.SplitWALManager(105): 122cddc95ec8,37369,1733206917509 WAL count=1, meta=true 2024-12-03T06:22:02,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta}] 2024-12-03T06:22:02,480 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T06:22:02,480 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:02,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466}] 2024-12-03T06:22:02,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37369-0x1009eb0b9ca0003, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T06:22:02,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37369-0x1009eb0b9ca0003, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T06:22:02,545 INFO [RS:2;122cddc95ec8:37369 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T06:22:02,545 INFO [RS:2;122cddc95ec8:37369 {}] regionserver.HRegionServer(1031): Exiting; stopping=122cddc95ec8,37369,1733206917509; zookeeper connection closed. 2024-12-03T06:22:02,545 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@78315e4d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@78315e4d 2024-12-03T06:22:02,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45577 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-12-03T06:22:02,675 INFO [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509-splitting/122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, size=31.4 K (32114bytes) 2024-12-03T06:22:02,676 DEBUG [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-03T06:22:02,676 DEBUG [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-03T06:22:02,676 INFO [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509-splitting/122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta 2024-12-03T06:22:02,680 INFO [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509-splitting/122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta after 3ms 2024-12-03T06:22:02,688 INFO [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509-splitting/122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta took 14ms 2024-12-03T06:22:02,698 DEBUG [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-12-03T06:22:02,703 INFO [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta.temp 2024-12-03T06:22:02,705 DEBUG [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509-splitting/122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta so closing down 2024-12-03T06:22:02,705 DEBUG [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T06:22:02,707 INFO [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta.temp 2024-12-03T06:22:02,710 INFO [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T06:22:02,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741864_1040 (size=31873) 2024-12-03T06:22:02,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741864_1040 (size=31873) 2024-12-03T06:22:02,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741864_1040 (size=31873) 2024-12-03T06:22:02,723 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) 2024-12-03T06:22:02,726 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta.temp to hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-12-03T06:22:02,729 INFO [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 36 ms; skipped=1; WAL=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509-splitting/122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false 2024-12-03T06:22:02,729 DEBUG [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509-splitting/122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, journal: Splitting hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509-splitting/122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, size=31.4 K (32114bytes) at 1733206922675Finishing writing output for hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509-splitting/122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta so closing down at 1733206922705 (+30 ms)Creating recovered edits writer path=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta.temp at 1733206922707 (+2 ms)3 split writer threads finished at 1733206922710 (+3 ms)Closed recovered edits writer path=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) at 1733206922723 (+13 ms)Rename recovered edits hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta.temp to hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1733206922726 (+3 ms)Processed 58 edits across 1 Regions in 36 ms; skipped=1; WAL=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509-splitting/122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false at 1733206922729 (+3 ms) 2024-12-03T06:22:02,730 DEBUG [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509-splitting/122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta 2024-12-03T06:22:02,731 DEBUG [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-12-03T06:22:02,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-12-03T06:22:02,742 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509-splitting/122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta to hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/oldWALs 2024-12-03T06:22:02,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-03T06:22:02,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 in 259 msec 2024-12-03T06:22:02,747 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:02,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-12-03T06:22:02,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 in 275 msec 2024-12-03T06:22:02,754 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509-splitting dir is empty, no logs to split. 2024-12-03T06:22:02,754 INFO [PEWorker-1 {}] master.SplitWALManager(105): 122cddc95ec8,37369,1733206917509 WAL count=0, meta=true 2024-12-03T06:22:02,754 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 122cddc95ec8,37369,1733206917509 WAL splitting is done? wals=0, meta=true 2024-12-03T06:22:02,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-03T06:22:02,759 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T06:22:02,761 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-03T06:22:02,911 DEBUG [122cddc95ec8:35025 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-03T06:22:02,912 DEBUG [122cddc95ec8:35025 {}] balancer.BalancerClusterState(204): Hosts are {122cddc95ec8=0} racks are {/default-rack=0} 2024-12-03T06:22:02,912 DEBUG [122cddc95ec8:35025 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T06:22:02,912 DEBUG [122cddc95ec8:35025 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T06:22:02,912 DEBUG [122cddc95ec8:35025 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T06:22:02,912 DEBUG [122cddc95ec8:35025 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T06:22:02,912 INFO [122cddc95ec8:35025 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T06:22:02,912 INFO [122cddc95ec8:35025 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T06:22:02,913 DEBUG [122cddc95ec8:35025 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T06:22:02,913 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:02,916 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 122cddc95ec8,45577,1733206917466, state=OPENING 2024-12-03T06:22:02,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:02,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:02,952 INFO [regionserver/122cddc95ec8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-03T06:22:02,952 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:02,952 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T06:22:02,952 INFO [regionserver/122cddc95ec8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-03T06:22:02,952 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:02,952 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:02,952 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:02,952 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466}] 2024-12-03T06:22:03,112 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-03T06:22:03,112 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T06:22:03,113 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-03T06:22:03,116 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=122cddc95ec8%2C45577%2C1733206917466.meta, suffix=.meta, logDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,45577,1733206917466, archiveDir=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/oldWALs, maxLogs=32 2024-12-03T06:22:03,129 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,45577,1733206917466/122cddc95ec8%2C45577%2C1733206917466.meta.1733206923118.meta, exclude list is [], retry=0 2024-12-03T06:22:03,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43645,DS-0c445287-33c1-4585-8daa-736a3c1373ca,DISK] 2024-12-03T06:22:03,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32933,DS-13cfb8bd-8a77-4bfd-914d-f7a5e73d8552,DISK] 2024-12-03T06:22:03,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40623,DS-e82fae88-53d6-4925-8312-e8210040c0a7,DISK] 2024-12-03T06:22:03,135 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,45577,1733206917466/122cddc95ec8%2C45577%2C1733206917466.meta.1733206923118.meta 2024-12-03T06:22:03,137 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41213:41213),(127.0.0.1/127.0.0.1:39137:39137),(127.0.0.1/127.0.0.1:36425:36425)] 2024-12-03T06:22:03,137 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-03T06:22:03,138 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-03T06:22:03,138 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-03T06:22:03,138 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-03T06:22:03,138 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-03T06:22:03,138 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:03,138 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-03T06:22:03,138 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-03T06:22:03,140 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T06:22:03,141 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T06:22:03,141 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:03,142 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T06:22:03,142 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T06:22:03,143 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T06:22:03,143 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:03,143 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T06:22:03,143 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T06:22:03,144 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T06:22:03,144 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:03,144 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T06:22:03,145 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T06:22:03,145 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T06:22:03,145 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:03,146 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T06:22:03,146 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T06:22:03,147 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740 2024-12-03T06:22:03,149 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740 2024-12-03T06:22:03,150 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-12-03T06:22:03,167 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-12-03T06:22:03,171 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=28 KB heapSize=48.31 KB 2024-12-03T06:22:03,234 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/.tmp/info/9723102379ca4611826c92221107d9b9 is 138, key is test,bbb,1733206919969.40b4b0967914401ebc70c24d4b56a381./info:regioninfo/1733206921145/Put/seqid=0 2024-12-03T06:22:03,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741866_1042 (size=34924) 2024-12-03T06:22:03,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741866_1042 (size=34924) 2024-12-03T06:22:03,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741866_1042 (size=34924) 2024-12-03T06:22:03,248 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.86 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/.tmp/info/9723102379ca4611826c92221107d9b9 2024-12-03T06:22:03,317 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/.tmp/ns/aacc1b2102a84e7a9a78500f1698215d is 43, key is default/ns:d/1733206919651/Put/seqid=0 2024-12-03T06:22:03,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741867_1043 (size=5153) 2024-12-03T06:22:03,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741867_1043 (size=5153) 2024-12-03T06:22:03,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741867_1043 (size=5153) 2024-12-03T06:22:03,326 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/.tmp/ns/aacc1b2102a84e7a9a78500f1698215d 2024-12-03T06:22:03,358 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/.tmp/table/d6e707759e5a45ee878d086852ae80f0 is 40, key is test/table:state/1733206921479/Put/seqid=0 2024-12-03T06:22:03,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741868_1044 (size=5165) 2024-12-03T06:22:03,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741868_1044 (size=5165) 2024-12-03T06:22:03,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741868_1044 (size=5165) 2024-12-03T06:22:03,367 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/.tmp/table/d6e707759e5a45ee878d086852ae80f0 2024-12-03T06:22:03,378 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/.tmp/info/9723102379ca4611826c92221107d9b9 as hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/info/9723102379ca4611826c92221107d9b9 2024-12-03T06:22:03,390 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/info/9723102379ca4611826c92221107d9b9, entries=260, sequenceid=60, filesize=34.1 K 2024-12-03T06:22:03,393 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/.tmp/ns/aacc1b2102a84e7a9a78500f1698215d as hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/ns/aacc1b2102a84e7a9a78500f1698215d 2024-12-03T06:22:03,402 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/ns/aacc1b2102a84e7a9a78500f1698215d, entries=2, sequenceid=60, filesize=5.0 K 2024-12-03T06:22:03,403 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/.tmp/table/d6e707759e5a45ee878d086852ae80f0 as hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/table/d6e707759e5a45ee878d086852ae80f0 2024-12-03T06:22:03,412 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/table/d6e707759e5a45ee878d086852ae80f0, entries=2, sequenceid=60, filesize=5.0 K 2024-12-03T06:22:03,416 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 245ms, sequenceid=60, compaction requested=false; wal=null 2024-12-03T06:22:03,418 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-03T06:22:03,421 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-12-03T06:22:03,422 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T06:22:03,422 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T06:22:03,423 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T06:22:03,425 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T06:22:03,428 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-12-03T06:22:03,429 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68850784, jitterRate=0.02595663070678711}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T06:22:03,429 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-03T06:22:03,430 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733206923138Writing region info on filesystem at 1733206923138Initializing all the Stores at 1733206923140 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733206923140Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733206923140Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206923140Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733206923140Obtaining lock to block concurrent updates at 1733206923171 (+31 ms)Preparing flush snapshotting stores in 1588230740 at 1733206923171Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=28672, getHeapSize=49408, getOffHeapSize=0, getCellsCount=264 at 1733206923174 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733206923174Flushing 1588230740/info: creating writer at 1733206923175 (+1 ms)Flushing 1588230740/info: appending metadata at 1733206923225 (+50 ms)Flushing 1588230740/info: closing flushed file at 1733206923227 (+2 ms)Flushing 1588230740/ns: creating writer at 1733206923295 (+68 ms)Flushing 1588230740/ns: appending metadata at 1733206923316 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1733206923316Flushing 1588230740/table: creating writer at 1733206923336 (+20 ms)Flushing 1588230740/table: appending metadata at 1733206923357 (+21 ms)Flushing 1588230740/table: closing flushed file at 1733206923357Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30dffc2d: reopening flushed file at 1733206923377 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4bdd605b: reopening flushed file at 1733206923392 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47210262: reopening flushed file at 1733206923402 (+10 ms)Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 245ms, sequenceid=60, compaction requested=false; wal=null at 1733206923417 (+15 ms)Cleaning up temporary data from old regions at 1733206923423 (+6 ms)Running coprocessor post-open hooks at 1733206923429 (+6 ms)Region opened successfully at 1733206923429 2024-12-03T06:22:03,431 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1733206923105 2024-12-03T06:22:03,434 DEBUG [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-03T06:22:03,434 INFO [RS_OPEN_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-03T06:22:03,435 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:03,437 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 122cddc95ec8,45577,1733206917466, state=OPEN 2024-12-03T06:22:03,467 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=122cddc95ec8,45577,1733206917466, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-03T06:22:03,571 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=122cddc95ec8,45577,1733206917466, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-03T06:22:03,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:03,599 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:03,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:03,599 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:03,599 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:03,599 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:03,600 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:03,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-03T06:22:03,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 in 647 msec 2024-12-03T06:22:03,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-12-03T06:22:03,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 846 msec 2024-12-03T06:22:03,607 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 122cddc95ec8,37369,1733206917509 had 9 regions 2024-12-03T06:22:03,608 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true, isMeta: false 2024-12-03T06:22:03,610 INFO [PEWorker-1 {}] master.SplitWALManager(105): 122cddc95ec8,37369,1733206917509 WAL count=1, meta=false 2024-12-03T06:22:03,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100}] 2024-12-03T06:22:03,612 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:03,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367}] 2024-12-03T06:22:03,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-12-03T06:22:03,787 INFO [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509-splitting/122cddc95ec8%2C37369%2C1733206917509.1733206919100, size=2.2 K (2232bytes) 2024-12-03T06:22:03,787 INFO [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509-splitting/122cddc95ec8%2C37369%2C1733206917509.1733206919100 2024-12-03T06:22:03,787 INFO [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509-splitting/122cddc95ec8%2C37369%2C1733206917509.1733206919100 after 0ms 2024-12-03T06:22:03,791 INFO [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509-splitting/122cddc95ec8%2C37369%2C1733206917509.1733206919100 took 5ms 2024-12-03T06:22:03,794 DEBUG [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 4e8636a0790f504025df15b4adf65735: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-03T06:22:03,796 DEBUG [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for ae2330e378b6ad4b1a3e345529a0b27f: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-03T06:22:03,798 DEBUG [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 923ece5c25680df02ff5f922d838bc62: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-03T06:22:03,800 DEBUG [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for e48f9dd0d4bf7a7f5e5fb4dac950f301: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-03T06:22:03,801 DEBUG [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 1ddf2a494d8d0fcd31eac685512f5644: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-03T06:22:03,803 DEBUG [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for a332f3da50b1f1fe35aa4e0307fe35f8: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-03T06:22:03,805 DEBUG [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 07dbe58fce62a9e98b77016cec645f5e: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-03T06:22:03,807 DEBUG [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 663ef4d3dbbcf7b1bb25bba1055da980: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-03T06:22:03,807 DEBUG [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509-splitting/122cddc95ec8%2C37369%2C1733206917509.1733206919100 so closing down 2024-12-03T06:22:03,807 DEBUG [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T06:22:03,807 INFO [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T06:22:03,808 INFO [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 8 edits across 0 Regions in 17 ms; skipped=8; WAL=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509-splitting/122cddc95ec8%2C37369%2C1733206917509.1733206919100, size=2.2 K, length=2232, corrupted=false, cancelled=false 2024-12-03T06:22:03,808 DEBUG [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509-splitting/122cddc95ec8%2C37369%2C1733206917509.1733206919100, journal: Splitting hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509-splitting/122cddc95ec8%2C37369%2C1733206917509.1733206919100, size=2.2 K (2232bytes) at 1733206923787Finishing writing output for hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509-splitting/122cddc95ec8%2C37369%2C1733206917509.1733206919100 so closing down at 1733206923807 (+20 ms)3 split writer threads finished at 1733206923808 (+1 ms)Processed 8 edits across 0 Regions in 17 ms; skipped=8; WAL=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509-splitting/122cddc95ec8%2C37369%2C1733206917509.1733206919100, size=2.2 K, length=2232, corrupted=false, cancelled=false at 1733206923808 2024-12-03T06:22:03,808 DEBUG [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509-splitting/122cddc95ec8%2C37369%2C1733206917509.1733206919100 2024-12-03T06:22:03,808 DEBUG [RS_LOG_REPLAY_OPS-regionserver/122cddc95ec8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-03T06:22:03,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35025 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-03T06:22:03,812 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509-splitting/122cddc95ec8%2C37369%2C1733206917509.1733206919100 to hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/oldWALs 2024-12-03T06:22:03,815 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-03T06:22:03,815 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 in 199 msec 2024-12-03T06:22:03,816 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:03,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-12-03T06:22:03,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 in 206 msec 2024-12-03T06:22:03,821 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/WALs/122cddc95ec8,37369,1733206917509-splitting dir is empty, no logs to split. 2024-12-03T06:22:03,821 INFO [PEWorker-1 {}] master.SplitWALManager(105): 122cddc95ec8,37369,1733206917509 WAL count=0, meta=false 2024-12-03T06:22:03,821 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 122cddc95ec8,37369,1733206917509 WAL splitting is done? wals=0, meta=false 2024-12-03T06:22:03,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN}] 2024-12-03T06:22:03,826 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN 2024-12-03T06:22:03,827 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-03T06:22:03,828 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN 2024-12-03T06:22:03,829 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-03T06:22:03,831 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN 2024-12-03T06:22:03,832 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-03T06:22:03,833 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN 2024-12-03T06:22:03,834 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-03T06:22:03,836 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN 2024-12-03T06:22:03,837 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-03T06:22:03,838 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:03,839 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-03T06:22:03,840 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:03,841 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-03T06:22:03,842 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:03,843 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-03T06:22:03,978 DEBUG [122cddc95ec8:35025 {}] balancer.BalancerClusterState(204): Hosts are {122cddc95ec8=0} racks are {/default-rack=0} 2024-12-03T06:22:03,978 DEBUG [122cddc95ec8:35025 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T06:22:03,978 DEBUG [122cddc95ec8:35025 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T06:22:03,978 DEBUG [122cddc95ec8:35025 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T06:22:03,978 DEBUG [122cddc95ec8:35025 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T06:22:03,979 INFO [122cddc95ec8:35025 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T06:22:03,979 INFO [122cddc95ec8:35025 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T06:22:03,979 DEBUG [122cddc95ec8:35025 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T06:22:03,979 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=923ece5c25680df02ff5f922d838bc62, regionState=OPENING, regionLocation=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:03,981 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=07dbe58fce62a9e98b77016cec645f5e, regionState=OPENING, regionLocation=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:03,982 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=a332f3da50b1f1fe35aa4e0307fe35f8, regionState=OPENING, regionLocation=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:03,982 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 122cddc95ec8:37369 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 122cddc95ec8/172.17.0.2:37369 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:22:03,984 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=663ef4d3dbbcf7b1bb25bba1055da980, regionState=OPENING, regionLocation=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:03,985 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=ae2330e378b6ad4b1a3e345529a0b27f, regionState=OPENING, regionLocation=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:03,986 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,37369,1733206917509, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,37369,1733206917509, seqNum=-1, error=java.net.ConnectException: Call to address=122cddc95ec8:37369 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 122cddc95ec8/172.17.0.2:37369 2024-12-03T06:22:03,986 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=4e8636a0790f504025df15b4adf65735, regionState=OPENING, regionLocation=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:03,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,37369,1733206917509, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-03T06:22:03,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,37369,1733206917509, seqNum=-1 from cache 2024-12-03T06:22:03,987 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=1ddf2a494d8d0fcd31eac685512f5644, regionState=OPENING, regionLocation=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:03,988 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T06:22:03,988 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,45577,1733206917466, seqNum=-1] 2024-12-03T06:22:03,988 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T06:22:03,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 122cddc95ec8:37369 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 122cddc95ec8/172.17.0.2:37369 2024-12-03T06:22:03,989 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=e48f9dd0d4bf7a7f5e5fb4dac950f301, regionState=OPENING, regionLocation=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:03,990 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 122cddc95ec8:37369 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 122cddc95ec8/172.17.0.2:37369 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:22:03,990 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,37369,1733206917509, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,45577,1733206917466, seqNum=-1, error=java.net.ConnectException: Call to address=122cddc95ec8:37369 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 122cddc95ec8/172.17.0.2:37369 2024-12-03T06:22:03,990 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,37369,1733206917509, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,45577,1733206917466, seqNum=-1, error=java.net.ConnectException: Call to address=122cddc95ec8:37369 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 122cddc95ec8/172.17.0.2:37369 2024-12-03T06:22:03,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,37369,1733206917509, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,45577,1733206917466, seqNum=-1, error=java.net.ConnectException: Call to address=122cddc95ec8:37369 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 122cddc95ec8/172.17.0.2:37369 2024-12-03T06:22:03,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,37369,1733206917509, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,45577,1733206917466, seqNum=-1, error=java.net.ConnectException: Call to address=122cddc95ec8:37369 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 122cddc95ec8/172.17.0.2:37369 2024-12-03T06:22:03,991 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54041, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T06:22:03,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,37369,1733206917509, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=122cddc95ec8,45577,1733206917466, seqNum=-1, error=java.net.ConnectException: Call to address=122cddc95ec8:37369 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 122cddc95ec8/172.17.0.2:37369 2024-12-03T06:22:03,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 122cddc95ec8:37369 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 122cddc95ec8/172.17.0.2:37369 2024-12-03T06:22:03,993 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN because future has completed 2024-12-03T06:22:03,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ddf2a494d8d0fcd31eac685512f5644, server=122cddc95ec8,43629,1733206917367}] 2024-12-03T06:22:03,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN because future has completed 2024-12-03T06:22:03,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367}] 2024-12-03T06:22:04,096 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN because future has completed 2024-12-03T06:22:04,096 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 923ece5c25680df02ff5f922d838bc62, server=122cddc95ec8,45577,1733206917466}] 2024-12-03T06:22:04,097 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN because future has completed 2024-12-03T06:22:04,098 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN because future has completed 2024-12-03T06:22:04,100 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN because future has completed 2024-12-03T06:22:04,101 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN because future has completed 2024-12-03T06:22:04,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae2330e378b6ad4b1a3e345529a0b27f, server=122cddc95ec8,43629,1733206917367}] 2024-12-03T06:22:04,102 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN because future has completed 2024-12-03T06:22:04,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e8636a0790f504025df15b4adf65735, server=122cddc95ec8,43629,1733206917367}] 2024-12-03T06:22:04,111 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 663ef4d3dbbcf7b1bb25bba1055da980, server=122cddc95ec8,45577,1733206917466}] 2024-12-03T06:22:04,115 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a332f3da50b1f1fe35aa4e0307fe35f8, server=122cddc95ec8,45577,1733206917466}] 2024-12-03T06:22:04,119 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07dbe58fce62a9e98b77016cec645f5e, server=122cddc95ec8,45577,1733206917466}] 2024-12-03T06:22:04,150 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open test,fff,1733206919969.e48f9dd0d4bf7a7f5e5fb4dac950f301. 2024-12-03T06:22:04,151 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => e48f9dd0d4bf7a7f5e5fb4dac950f301, NAME => 'test,fff,1733206919969.e48f9dd0d4bf7a7f5e5fb4dac950f301.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-12-03T06:22:04,151 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e48f9dd0d4bf7a7f5e5fb4dac950f301 2024-12-03T06:22:04,151 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated test,fff,1733206919969.e48f9dd0d4bf7a7f5e5fb4dac950f301.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:04,151 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for e48f9dd0d4bf7a7f5e5fb4dac950f301 2024-12-03T06:22:04,151 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for e48f9dd0d4bf7a7f5e5fb4dac950f301 2024-12-03T06:22:04,153 INFO [StoreOpener-e48f9dd0d4bf7a7f5e5fb4dac950f301-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e48f9dd0d4bf7a7f5e5fb4dac950f301 2024-12-03T06:22:04,154 INFO [StoreOpener-e48f9dd0d4bf7a7f5e5fb4dac950f301-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e48f9dd0d4bf7a7f5e5fb4dac950f301 columnFamilyName family 2024-12-03T06:22:04,154 DEBUG [StoreOpener-e48f9dd0d4bf7a7f5e5fb4dac950f301-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:04,154 INFO [StoreOpener-e48f9dd0d4bf7a7f5e5fb4dac950f301-1 {}] regionserver.HStore(327): Store=e48f9dd0d4bf7a7f5e5fb4dac950f301/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:04,154 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for e48f9dd0d4bf7a7f5e5fb4dac950f301 2024-12-03T06:22:04,155 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/e48f9dd0d4bf7a7f5e5fb4dac950f301 2024-12-03T06:22:04,157 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/e48f9dd0d4bf7a7f5e5fb4dac950f301 2024-12-03T06:22:04,157 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for e48f9dd0d4bf7a7f5e5fb4dac950f301 2024-12-03T06:22:04,157 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for e48f9dd0d4bf7a7f5e5fb4dac950f301 2024-12-03T06:22:04,159 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for e48f9dd0d4bf7a7f5e5fb4dac950f301 2024-12-03T06:22:04,160 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened e48f9dd0d4bf7a7f5e5fb4dac950f301; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65178542, jitterRate=-0.028764039278030396}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:04,160 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e48f9dd0d4bf7a7f5e5fb4dac950f301 2024-12-03T06:22:04,160 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for e48f9dd0d4bf7a7f5e5fb4dac950f301: Running coprocessor pre-open hook at 1733206924151Writing region info on filesystem at 1733206924151Initializing all the Stores at 1733206924152 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206924152Cleaning up temporary data from old regions at 1733206924157 (+5 ms)Running coprocessor post-open hooks at 1733206924160 (+3 ms)Region opened successfully at 1733206924160 2024-12-03T06:22:04,161 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1733206919969.e48f9dd0d4bf7a7f5e5fb4dac950f301., pid=73, masterSystemTime=1733206924146 2024-12-03T06:22:04,164 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1733206919969.e48f9dd0d4bf7a7f5e5fb4dac950f301. 2024-12-03T06:22:04,164 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened test,fff,1733206919969.e48f9dd0d4bf7a7f5e5fb4dac950f301. 2024-12-03T06:22:04,164 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,sss,1733206919969.4e8636a0790f504025df15b4adf65735. 2024-12-03T06:22:04,164 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 4e8636a0790f504025df15b4adf65735, NAME => 'test,sss,1733206919969.4e8636a0790f504025df15b4adf65735.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-12-03T06:22:04,165 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4e8636a0790f504025df15b4adf65735 2024-12-03T06:22:04,165 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,sss,1733206919969.4e8636a0790f504025df15b4adf65735.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:04,165 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 4e8636a0790f504025df15b4adf65735 2024-12-03T06:22:04,165 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 4e8636a0790f504025df15b4adf65735 2024-12-03T06:22:04,167 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-12-03T06:22:04,167 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,167 INFO [StoreOpener-4e8636a0790f504025df15b4adf65735-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4e8636a0790f504025df15b4adf65735 2024-12-03T06:22:04,167 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,168 INFO [StoreOpener-4e8636a0790f504025df15b4adf65735-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e8636a0790f504025df15b4adf65735 columnFamilyName family 2024-12-03T06:22:04,169 DEBUG [StoreOpener-4e8636a0790f504025df15b4adf65735-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:04,169 INFO [StoreOpener-4e8636a0790f504025df15b4adf65735-1 {}] regionserver.HStore(327): Store=4e8636a0790f504025df15b4adf65735/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:04,169 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 4e8636a0790f504025df15b4adf65735 2024-12-03T06:22:04,170 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/4e8636a0790f504025df15b4adf65735 2024-12-03T06:22:04,172 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/4e8636a0790f504025df15b4adf65735 2024-12-03T06:22:04,172 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 4e8636a0790f504025df15b4adf65735 2024-12-03T06:22:04,173 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 4e8636a0790f504025df15b4adf65735 2024-12-03T06:22:04,175 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 4e8636a0790f504025df15b4adf65735 2024-12-03T06:22:04,176 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 4e8636a0790f504025df15b4adf65735; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75423189, jitterRate=0.12389309704303741}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:04,176 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4e8636a0790f504025df15b4adf65735 2024-12-03T06:22:04,177 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 4e8636a0790f504025df15b4adf65735: Running coprocessor pre-open hook at 1733206924165Writing region info on filesystem at 1733206924165Initializing all the Stores at 1733206924167 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206924167Cleaning up temporary data from old regions at 1733206924173 (+6 ms)Running coprocessor post-open hooks at 1733206924176 (+3 ms)Region opened successfully at 1733206924176 2024-12-03T06:22:04,177 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1733206919969.4e8636a0790f504025df15b4adf65735., pid=76, masterSystemTime=1733206924146 2024-12-03T06:22:04,180 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1733206919969.4e8636a0790f504025df15b4adf65735. 2024-12-03T06:22:04,180 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,sss,1733206919969.4e8636a0790f504025df15b4adf65735. 2024-12-03T06:22:04,180 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,yyy,1733206919969.ae2330e378b6ad4b1a3e345529a0b27f. 2024-12-03T06:22:04,179 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-12-03T06:22:04,180 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => ae2330e378b6ad4b1a3e345529a0b27f, NAME => 'test,yyy,1733206919969.ae2330e378b6ad4b1a3e345529a0b27f.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-12-03T06:22:04,181 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ae2330e378b6ad4b1a3e345529a0b27f 2024-12-03T06:22:04,181 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,yyy,1733206919969.ae2330e378b6ad4b1a3e345529a0b27f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:04,181 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T06:22:04,181 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07dbe58fce62a9e98b77016cec645f5e, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,181 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for ae2330e378b6ad4b1a3e345529a0b27f 2024-12-03T06:22:04,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,181 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for ae2330e378b6ad4b1a3e345529a0b27f 2024-12-03T06:22:04,183 INFO [StoreOpener-ae2330e378b6ad4b1a3e345529a0b27f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ae2330e378b6ad4b1a3e345529a0b27f 2024-12-03T06:22:04,184 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T06:22:04,184 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,184 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T06:22:04,184 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T06:22:04,185 INFO [StoreOpener-ae2330e378b6ad4b1a3e345529a0b27f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae2330e378b6ad4b1a3e345529a0b27f columnFamilyName family 2024-12-03T06:22:04,184 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07dbe58fce62a9e98b77016cec645f5e, server=122cddc95ec8,45577,1733206917466 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,185 DEBUG [StoreOpener-ae2330e378b6ad4b1a3e345529a0b27f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:04,185 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T06:22:04,185 INFO [StoreOpener-ae2330e378b6ad4b1a3e345529a0b27f-1 {}] regionserver.HStore(327): Store=ae2330e378b6ad4b1a3e345529a0b27f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:04,185 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T06:22:04,185 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T06:22:04,185 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for ae2330e378b6ad4b1a3e345529a0b27f 2024-12-03T06:22:04,186 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T06:22:04,186 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T06:22:04,186 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/ae2330e378b6ad4b1a3e345529a0b27f 2024-12-03T06:22:04,188 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/ae2330e378b6ad4b1a3e345529a0b27f 2024-12-03T06:22:04,189 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for ae2330e378b6ad4b1a3e345529a0b27f 2024-12-03T06:22:04,189 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for ae2330e378b6ad4b1a3e345529a0b27f 2024-12-03T06:22:04,191 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for ae2330e378b6ad4b1a3e345529a0b27f 2024-12-03T06:22:04,192 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T06:22:04,193 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened ae2330e378b6ad4b1a3e345529a0b27f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65098956, jitterRate=-0.029949963092803955}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:04,193 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ae2330e378b6ad4b1a3e345529a0b27f 2024-12-03T06:22:04,193 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for ae2330e378b6ad4b1a3e345529a0b27f: Running coprocessor pre-open hook at 1733206924181Writing region info on filesystem at 1733206924181Initializing all the Stores at 1733206924183 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206924183Cleaning up temporary data from old regions at 1733206924189 (+6 ms)Running coprocessor post-open hooks at 1733206924193 (+4 ms)Region opened successfully at 1733206924193 2024-12-03T06:22:04,193 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,193 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,193 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,193 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,193 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,193 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,193 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,193 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,194 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,194 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,194 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,194 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,194 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,194 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,194 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,194 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,194 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,194 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1733206919969.ae2330e378b6ad4b1a3e345529a0b27f., pid=75, masterSystemTime=1733206924146 2024-12-03T06:22:04,194 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,194 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,194 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,194 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,194 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,195 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,195 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T06:22:04,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T06:22:04,196 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "122cddc95ec8" port: 43629 start_code: 1733206917367 } transition { transition_code: OPENED region_info { region_id: 1733206919969 table_name { namespace: "default" qualifier: "test" } start_key: "yyy" end_key: "zzz" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 75 initiating_master_active_time: 1733206917630 }; retry (#0) after 1005ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-12-03T06:22:04,198 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,202 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true 2024-12-03T06:22:04,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T06:22:04,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:04,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN 2024-12-03T06:22:04,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN 2024-12-03T06:22:04,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN 2024-12-03T06:22:04,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN 2024-12-03T06:22:04,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN 2024-12-03T06:22:04,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ddf2a494d8d0fcd31eac685512f5644, server=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 923ece5c25680df02ff5f922d838bc62, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae2330e378b6ad4b1a3e345529a0b27f, server=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e8636a0790f504025df15b4adf65735, server=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 663ef4d3dbbcf7b1bb25bba1055da980, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a332f3da50b1f1fe35aa4e0307fe35f8, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07dbe58fce62a9e98b77016cec645f5e, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,206 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,206 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,206 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,207 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,207 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,207 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,207 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,207 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,207 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,207 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,207 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,207 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,207 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:04,207 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,207 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN 2024-12-03T06:22:04,207 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,207 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN 2024-12-03T06:22:04,207 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,207 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN 2024-12-03T06:22:04,207 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,207 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN 2024-12-03T06:22:04,207 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,207 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN 2024-12-03T06:22:04,207 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ddf2a494d8d0fcd31eac685512f5644, server=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,207 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,207 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 923ece5c25680df02ff5f922d838bc62, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,207 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae2330e378b6ad4b1a3e345529a0b27f, server=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,208 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e8636a0790f504025df15b4adf65735, server=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,208 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 663ef4d3dbbcf7b1bb25bba1055da980, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,208 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a332f3da50b1f1fe35aa4e0307fe35f8, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,208 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07dbe58fce62a9e98b77016cec645f5e, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,208 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 21 msec 2024-12-03T06:22:04,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=61 2024-12-03T06:22:04,211 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=122cddc95ec8,45577,1733206917466, table=hbase:meta, region=1588230740 2024-12-03T06:22:04,211 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 122cddc95ec8,45577,1733206917466, state=OPEN 2024-12-03T06:22:04,252 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,,1733206919969.663ef4d3dbbcf7b1bb25bba1055da980. 2024-12-03T06:22:04,253 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 663ef4d3dbbcf7b1bb25bba1055da980, NAME => 'test,,1733206919969.663ef4d3dbbcf7b1bb25bba1055da980.', STARTKEY => '', ENDKEY => 'bbb'} 2024-12-03T06:22:04,253 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 663ef4d3dbbcf7b1bb25bba1055da980 2024-12-03T06:22:04,253 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,,1733206919969.663ef4d3dbbcf7b1bb25bba1055da980.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:04,253 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 663ef4d3dbbcf7b1bb25bba1055da980 2024-12-03T06:22:04,253 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 663ef4d3dbbcf7b1bb25bba1055da980 2024-12-03T06:22:04,255 INFO [StoreOpener-663ef4d3dbbcf7b1bb25bba1055da980-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 663ef4d3dbbcf7b1bb25bba1055da980 2024-12-03T06:22:04,256 INFO [StoreOpener-663ef4d3dbbcf7b1bb25bba1055da980-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 663ef4d3dbbcf7b1bb25bba1055da980 columnFamilyName family 2024-12-03T06:22:04,256 DEBUG [StoreOpener-663ef4d3dbbcf7b1bb25bba1055da980-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:04,257 INFO [StoreOpener-663ef4d3dbbcf7b1bb25bba1055da980-1 {}] regionserver.HStore(327): Store=663ef4d3dbbcf7b1bb25bba1055da980/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:04,257 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 663ef4d3dbbcf7b1bb25bba1055da980 2024-12-03T06:22:04,258 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/663ef4d3dbbcf7b1bb25bba1055da980 2024-12-03T06:22:04,260 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/663ef4d3dbbcf7b1bb25bba1055da980 2024-12-03T06:22:04,260 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 663ef4d3dbbcf7b1bb25bba1055da980 2024-12-03T06:22:04,261 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 663ef4d3dbbcf7b1bb25bba1055da980 2024-12-03T06:22:04,263 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 663ef4d3dbbcf7b1bb25bba1055da980 2024-12-03T06:22:04,264 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 663ef4d3dbbcf7b1bb25bba1055da980; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67029880, jitterRate=-0.0011769533157348633}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:04,264 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 663ef4d3dbbcf7b1bb25bba1055da980 2024-12-03T06:22:04,264 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 663ef4d3dbbcf7b1bb25bba1055da980: Running coprocessor pre-open hook at 1733206924253Writing region info on filesystem at 1733206924253Initializing all the Stores at 1733206924254 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206924255 (+1 ms)Cleaning up temporary data from old regions at 1733206924261 (+6 ms)Running coprocessor post-open hooks at 1733206924264 (+3 ms)Region opened successfully at 1733206924264 2024-12-03T06:22:04,266 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1733206919969.663ef4d3dbbcf7b1bb25bba1055da980., pid=77, masterSystemTime=1733206924248 2024-12-03T06:22:04,267 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,268 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "122cddc95ec8" port: 45577 start_code: 1733206917466 } transition { transition_code: OPENED region_info { region_id: 1733206919969 table_name { namespace: "default" qualifier: "test" } start_key: "" end_key: "bbb" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 77 initiating_master_active_time: 1733206917630 }; retry (#0) after 1001ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-12-03T06:22:04,304 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,309 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,310 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,311 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN to state=OFFLINE, location=null, table=test, region=4e8636a0790f504025df15b4adf65735 to restore RIT 2024-12-03T06:22:04,312 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN to state=OFFLINE, location=null, table=test, region=ae2330e378b6ad4b1a3e345529a0b27f to restore RIT 2024-12-03T06:22:04,312 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN to state=OFFLINE, location=null, table=test, region=1ddf2a494d8d0fcd31eac685512f5644 to restore RIT 2024-12-03T06:22:04,312 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN to state=OFFLINE, location=null, table=test, region=663ef4d3dbbcf7b1bb25bba1055da980 to restore RIT 2024-12-03T06:22:04,312 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN to state=OFFLINE, location=null, table=test, region=923ece5c25680df02ff5f922d838bc62 to restore RIT 2024-12-03T06:22:04,312 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN to state=OFFLINE, location=null, table=test, region=07dbe58fce62a9e98b77016cec645f5e to restore RIT 2024-12-03T06:22:04,312 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN to state=OFFLINE, location=null, table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8 to restore RIT 2024-12-03T06:22:04,312 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN to state=OFFLINE, location=null, table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301 to restore RIT 2024-12-03T06:22:04,312 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733206954312 2024-12-03T06:22:04,313 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T06:22:04,313 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,314 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T06:22:04,315 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T06:22:04,316 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a332f3da50b1f1fe35aa4e0307fe35f8, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,316 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a332f3da50b1f1fe35aa4e0307fe35f8, server=122cddc95ec8,45577,1733206917466 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663ef4d3dbbcf7b1bb25bba1055da980, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b4b0967914401ebc70c24d4b56a381, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bbc9d78d890c8d774a80646d9983ed6, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=33124cac09cf3a2502ce5e473bf614f4, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e884340ece2e3eb17a321ed6d089e5e7, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e48f9dd0d4bf7a7f5e5fb4dac950f301, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca916e60e459eeefe786a877b8700115, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d719355720f98b8bc93f39a6130075b, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a332f3da50b1f1fe35aa4e0307fe35f8, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c9e860d892b47160f56d24cb24e6010, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f495460f3e799a3e64378f8879c721bb, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ddf2a494d8d0fcd31eac685512f5644, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=050ab97be4cc9f90a155d3b261335a0d, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5f7ab205e6c897503e3ec729aaa3170, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6fcc2e40df410122347a4873dfd1129, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07dbe58fce62a9e98b77016cec645f5e, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6badf4f9ef3213a92a5f6ace694c6482, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afa4ab371a599cc21eb81082233626e3, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8636a0790f504025df15b4adf65735, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e061778c0efa365f63836d0691ff7a8, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3e1e4a86d09a1acb1fcf9d92f129898, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923ece5c25680df02ff5f922d838bc62, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9f6e863b0ed7a188cb723f7a298ba2c, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b148e6fa0896eea84e6c38d5293e44a, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae2330e378b6ad4b1a3e345529a0b27f, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=671297fbd74543fd2b2deed06d08e6a7, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,350 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T06:22:04,350 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733206984350 2024-12-03T06:22:04,351 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733207044351 2024-12-03T06:22:04,351 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 38 msec 2024-12-03T06:22:04,351 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T06:22:04,351 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,351 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T06:22:04,351 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T06:22:04,351 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T06:22:04,351 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T06:22:04,351 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T06:22:04,351 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T06:22:04,352 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T06:22:04,356 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T06:22:04,356 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,356 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,356 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,356 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,356 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,356 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,356 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,356 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,356 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,356 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,357 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,357 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,357 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,357 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,357 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,357 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,357 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,357 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,357 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,357 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,357 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T06:22:04,358 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T06:22:04,360 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true 2024-12-03T06:22:04,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T06:22:04,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:04,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN 2024-12-03T06:22:04,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN 2024-12-03T06:22:04,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN 2024-12-03T06:22:04,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN 2024-12-03T06:22:04,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN 2024-12-03T06:22:04,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ddf2a494d8d0fcd31eac685512f5644, server=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 923ece5c25680df02ff5f922d838bc62, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae2330e378b6ad4b1a3e345529a0b27f, server=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e8636a0790f504025df15b4adf65735, server=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 663ef4d3dbbcf7b1bb25bba1055da980, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a332f3da50b1f1fe35aa4e0307fe35f8, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,361 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,361 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,361 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,361 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,361 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,361 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,361 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,361 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,362 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,362 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,362 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,362 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,362 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:04,362 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,362 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN 2024-12-03T06:22:04,362 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,362 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN 2024-12-03T06:22:04,362 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,362 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN 2024-12-03T06:22:04,362 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,362 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN 2024-12-03T06:22:04,362 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,362 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN 2024-12-03T06:22:04,362 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ddf2a494d8d0fcd31eac685512f5644, server=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,362 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,362 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 923ece5c25680df02ff5f922d838bc62, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,362 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae2330e378b6ad4b1a3e345529a0b27f, server=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,362 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e8636a0790f504025df15b4adf65735, server=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,362 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 663ef4d3dbbcf7b1bb25bba1055da980, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,362 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a332f3da50b1f1fe35aa4e0307fe35f8, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,362 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-12-03T06:22:04,363 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=61 2024-12-03T06:22:04,363 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=122cddc95ec8,45577,1733206917466, table=hbase:meta, region=1588230740 2024-12-03T06:22:04,364 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 122cddc95ec8,45577,1733206917466, state=OPEN 2024-12-03T06:22:04,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,382 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,382 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,382 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,382 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,383 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN to state=OFFLINE, location=null, table=test, region=4e8636a0790f504025df15b4adf65735 to restore RIT 2024-12-03T06:22:04,383 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN to state=OFFLINE, location=null, table=test, region=ae2330e378b6ad4b1a3e345529a0b27f to restore RIT 2024-12-03T06:22:04,383 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN to state=OFFLINE, location=null, table=test, region=1ddf2a494d8d0fcd31eac685512f5644 to restore RIT 2024-12-03T06:22:04,383 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN to state=OFFLINE, location=null, table=test, region=663ef4d3dbbcf7b1bb25bba1055da980 to restore RIT 2024-12-03T06:22:04,383 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN to state=OFFLINE, location=null, table=test, region=923ece5c25680df02ff5f922d838bc62 to restore RIT 2024-12-03T06:22:04,383 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN to state=OFFLINE, location=null, table=test, region=07dbe58fce62a9e98b77016cec645f5e to restore RIT 2024-12-03T06:22:04,383 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN to state=OFFLINE, location=null, table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8 to restore RIT 2024-12-03T06:22:04,383 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN to state=OFFLINE, location=null, table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301 to restore RIT 2024-12-03T06:22:04,384 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733206954384 2024-12-03T06:22:04,384 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T06:22:04,384 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,386 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T06:22:04,387 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T06:22:04,387 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 663ef4d3dbbcf7b1bb25bba1055da980, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,388 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,388 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 663ef4d3dbbcf7b1bb25bba1055da980, server=122cddc95ec8,45577,1733206917466 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,394 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663ef4d3dbbcf7b1bb25bba1055da980, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,395 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b4b0967914401ebc70c24d4b56a381, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,395 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bbc9d78d890c8d774a80646d9983ed6, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,395 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=33124cac09cf3a2502ce5e473bf614f4, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,395 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e884340ece2e3eb17a321ed6d089e5e7, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,396 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e48f9dd0d4bf7a7f5e5fb4dac950f301, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,396 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca916e60e459eeefe786a877b8700115, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,396 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d719355720f98b8bc93f39a6130075b, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,396 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a332f3da50b1f1fe35aa4e0307fe35f8, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,396 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c9e860d892b47160f56d24cb24e6010, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f495460f3e799a3e64378f8879c721bb, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ddf2a494d8d0fcd31eac685512f5644, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=050ab97be4cc9f90a155d3b261335a0d, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5f7ab205e6c897503e3ec729aaa3170, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6fcc2e40df410122347a4873dfd1129, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07dbe58fce62a9e98b77016cec645f5e, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6badf4f9ef3213a92a5f6ace694c6482, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afa4ab371a599cc21eb81082233626e3, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8636a0790f504025df15b4adf65735, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e061778c0efa365f63836d0691ff7a8, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3e1e4a86d09a1acb1fcf9d92f129898, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923ece5c25680df02ff5f922d838bc62, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9f6e863b0ed7a188cb723f7a298ba2c, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b148e6fa0896eea84e6c38d5293e44a, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae2330e378b6ad4b1a3e345529a0b27f, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=671297fbd74543fd2b2deed06d08e6a7, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,400 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T06:22:04,400 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733206984400 2024-12-03T06:22:04,400 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733207044400 2024-12-03T06:22:04,400 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 16 msec 2024-12-03T06:22:04,400 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T06:22:04,400 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,400 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T06:22:04,400 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T06:22:04,401 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T06:22:04,401 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T06:22:04,401 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T06:22:04,401 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T06:22:04,402 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T06:22:04,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T06:22:04,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,408 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,408 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,408 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,408 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,408 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,408 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T06:22:04,409 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T06:22:04,412 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true 2024-12-03T06:22:04,412 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,412 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,412 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T06:22:04,412 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,412 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,412 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,412 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,412 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,412 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:04,413 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN 2024-12-03T06:22:04,413 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN 2024-12-03T06:22:04,413 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN 2024-12-03T06:22:04,412 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,413 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN 2024-12-03T06:22:04,413 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN 2024-12-03T06:22:04,413 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ddf2a494d8d0fcd31eac685512f5644, server=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,413 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,413 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 923ece5c25680df02ff5f922d838bc62, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,413 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae2330e378b6ad4b1a3e345529a0b27f, server=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,413 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e8636a0790f504025df15b4adf65735, server=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,413 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 663ef4d3dbbcf7b1bb25bba1055da980, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,413 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,413 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,413 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,413 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,413 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,414 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,414 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,414 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,414 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,414 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,414 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,414 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,414 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,414 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:04,414 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,414 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN 2024-12-03T06:22:04,414 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,414 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN 2024-12-03T06:22:04,414 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,414 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN 2024-12-03T06:22:04,414 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,414 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN 2024-12-03T06:22:04,415 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,415 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN 2024-12-03T06:22:04,415 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ddf2a494d8d0fcd31eac685512f5644, server=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,415 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,415 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 923ece5c25680df02ff5f922d838bc62, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,415 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae2330e378b6ad4b1a3e345529a0b27f, server=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,415 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e8636a0790f504025df15b4adf65735, server=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,415 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 663ef4d3dbbcf7b1bb25bba1055da980, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,415 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 12 msec 2024-12-03T06:22:04,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=61 2024-12-03T06:22:04,417 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=122cddc95ec8,45577,1733206917466, table=hbase:meta, region=1588230740 2024-12-03T06:22:04,417 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 122cddc95ec8,45577,1733206917466, state=OPEN 2024-12-03T06:22:04,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,427 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,427 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,427 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,427 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,428 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN to state=OFFLINE, location=null, table=test, region=4e8636a0790f504025df15b4adf65735 to restore RIT 2024-12-03T06:22:04,428 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN to state=OFFLINE, location=null, table=test, region=ae2330e378b6ad4b1a3e345529a0b27f to restore RIT 2024-12-03T06:22:04,428 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN to state=OFFLINE, location=null, table=test, region=1ddf2a494d8d0fcd31eac685512f5644 to restore RIT 2024-12-03T06:22:04,428 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN to state=OFFLINE, location=null, table=test, region=663ef4d3dbbcf7b1bb25bba1055da980 to restore RIT 2024-12-03T06:22:04,428 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN to state=OFFLINE, location=null, table=test, region=923ece5c25680df02ff5f922d838bc62 to restore RIT 2024-12-03T06:22:04,428 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN to state=OFFLINE, location=null, table=test, region=07dbe58fce62a9e98b77016cec645f5e to restore RIT 2024-12-03T06:22:04,428 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN to state=OFFLINE, location=null, table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8 to restore RIT 2024-12-03T06:22:04,428 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN to state=OFFLINE, location=null, table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301 to restore RIT 2024-12-03T06:22:04,429 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733206954429 2024-12-03T06:22:04,429 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T06:22:04,429 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,431 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T06:22:04,432 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T06:22:04,432 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e8636a0790f504025df15b4adf65735, server=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,433 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e8636a0790f504025df15b4adf65735, server=122cddc95ec8,43629,1733206917367 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663ef4d3dbbcf7b1bb25bba1055da980, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b4b0967914401ebc70c24d4b56a381, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bbc9d78d890c8d774a80646d9983ed6, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=33124cac09cf3a2502ce5e473bf614f4, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e884340ece2e3eb17a321ed6d089e5e7, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e48f9dd0d4bf7a7f5e5fb4dac950f301, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca916e60e459eeefe786a877b8700115, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d719355720f98b8bc93f39a6130075b, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a332f3da50b1f1fe35aa4e0307fe35f8, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c9e860d892b47160f56d24cb24e6010, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f495460f3e799a3e64378f8879c721bb, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ddf2a494d8d0fcd31eac685512f5644, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=050ab97be4cc9f90a155d3b261335a0d, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5f7ab205e6c897503e3ec729aaa3170, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6fcc2e40df410122347a4873dfd1129, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07dbe58fce62a9e98b77016cec645f5e, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6badf4f9ef3213a92a5f6ace694c6482, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afa4ab371a599cc21eb81082233626e3, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8636a0790f504025df15b4adf65735, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e061778c0efa365f63836d0691ff7a8, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3e1e4a86d09a1acb1fcf9d92f129898, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923ece5c25680df02ff5f922d838bc62, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9f6e863b0ed7a188cb723f7a298ba2c, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b148e6fa0896eea84e6c38d5293e44a, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae2330e378b6ad4b1a3e345529a0b27f, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=671297fbd74543fd2b2deed06d08e6a7, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,440 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T06:22:04,440 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733206984440 2024-12-03T06:22:04,440 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733207044440 2024-12-03T06:22:04,440 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-12-03T06:22:04,440 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T06:22:04,440 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,440 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T06:22:04,440 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T06:22:04,440 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T06:22:04,440 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T06:22:04,440 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T06:22:04,440 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T06:22:04,441 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T06:22:04,444 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T06:22:04,444 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,444 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,444 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,444 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,444 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,444 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,444 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,444 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,445 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,445 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,445 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,445 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,445 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,445 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,445 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,445 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,445 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,445 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,445 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,445 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T06:22:04,446 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T06:22:04,448 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true 2024-12-03T06:22:04,448 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,448 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,448 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T06:22:04,448 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,448 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,448 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,448 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,448 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,448 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:04,448 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN 2024-12-03T06:22:04,448 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN 2024-12-03T06:22:04,448 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN 2024-12-03T06:22:04,448 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN 2024-12-03T06:22:04,448 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN 2024-12-03T06:22:04,448 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ddf2a494d8d0fcd31eac685512f5644, server=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 923ece5c25680df02ff5f922d838bc62, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae2330e378b6ad4b1a3e345529a0b27f, server=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e8636a0790f504025df15b4adf65735, server=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,449 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,449 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,449 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,449 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,449 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,449 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,449 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,449 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,449 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,449 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,449 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,449 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,449 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:04,449 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,449 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN 2024-12-03T06:22:04,449 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,449 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN 2024-12-03T06:22:04,449 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,449 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN 2024-12-03T06:22:04,449 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,449 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN 2024-12-03T06:22:04,449 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,450 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN 2024-12-03T06:22:04,450 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ddf2a494d8d0fcd31eac685512f5644, server=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,450 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,450 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 923ece5c25680df02ff5f922d838bc62, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,450 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae2330e378b6ad4b1a3e345529a0b27f, server=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,450 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e8636a0790f504025df15b4adf65735, server=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,450 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-12-03T06:22:04,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=61 2024-12-03T06:22:04,451 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=122cddc95ec8,45577,1733206917466, table=hbase:meta, region=1588230740 2024-12-03T06:22:04,451 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 122cddc95ec8,45577,1733206917466, state=OPEN 2024-12-03T06:22:04,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,460 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,461 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,461 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,461 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,461 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN to state=OFFLINE, location=null, table=test, region=4e8636a0790f504025df15b4adf65735 to restore RIT 2024-12-03T06:22:04,461 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN to state=OFFLINE, location=null, table=test, region=ae2330e378b6ad4b1a3e345529a0b27f to restore RIT 2024-12-03T06:22:04,461 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN to state=OFFLINE, location=null, table=test, region=1ddf2a494d8d0fcd31eac685512f5644 to restore RIT 2024-12-03T06:22:04,461 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN to state=OFFLINE, location=null, table=test, region=663ef4d3dbbcf7b1bb25bba1055da980 to restore RIT 2024-12-03T06:22:04,462 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN to state=OFFLINE, location=null, table=test, region=923ece5c25680df02ff5f922d838bc62 to restore RIT 2024-12-03T06:22:04,462 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN to state=OFFLINE, location=null, table=test, region=07dbe58fce62a9e98b77016cec645f5e to restore RIT 2024-12-03T06:22:04,462 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN to state=OFFLINE, location=null, table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8 to restore RIT 2024-12-03T06:22:04,462 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN to state=OFFLINE, location=null, table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301 to restore RIT 2024-12-03T06:22:04,462 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733206954462 2024-12-03T06:22:04,462 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T06:22:04,462 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,464 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T06:22:04,466 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T06:22:04,466 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae2330e378b6ad4b1a3e345529a0b27f, server=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,466 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae2330e378b6ad4b1a3e345529a0b27f, server=122cddc95ec8,43629,1733206917367 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663ef4d3dbbcf7b1bb25bba1055da980, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b4b0967914401ebc70c24d4b56a381, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bbc9d78d890c8d774a80646d9983ed6, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=33124cac09cf3a2502ce5e473bf614f4, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e884340ece2e3eb17a321ed6d089e5e7, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e48f9dd0d4bf7a7f5e5fb4dac950f301, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca916e60e459eeefe786a877b8700115, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d719355720f98b8bc93f39a6130075b, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a332f3da50b1f1fe35aa4e0307fe35f8, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c9e860d892b47160f56d24cb24e6010, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f495460f3e799a3e64378f8879c721bb, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ddf2a494d8d0fcd31eac685512f5644, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=050ab97be4cc9f90a155d3b261335a0d, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5f7ab205e6c897503e3ec729aaa3170, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6fcc2e40df410122347a4873dfd1129, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07dbe58fce62a9e98b77016cec645f5e, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6badf4f9ef3213a92a5f6ace694c6482, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afa4ab371a599cc21eb81082233626e3, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8636a0790f504025df15b4adf65735, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e061778c0efa365f63836d0691ff7a8, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3e1e4a86d09a1acb1fcf9d92f129898, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923ece5c25680df02ff5f922d838bc62, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9f6e863b0ed7a188cb723f7a298ba2c, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b148e6fa0896eea84e6c38d5293e44a, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae2330e378b6ad4b1a3e345529a0b27f, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=671297fbd74543fd2b2deed06d08e6a7, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,473 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T06:22:04,473 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733206984473 2024-12-03T06:22:04,473 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733207044473 2024-12-03T06:22:04,473 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-12-03T06:22:04,473 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T06:22:04,473 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,473 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T06:22:04,473 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T06:22:04,474 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T06:22:04,474 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T06:22:04,474 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T06:22:04,474 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T06:22:04,475 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T06:22:04,477 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T06:22:04,477 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,478 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T06:22:04,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T06:22:04,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true 2024-12-03T06:22:04,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T06:22:04,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:04,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN 2024-12-03T06:22:04,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN 2024-12-03T06:22:04,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN 2024-12-03T06:22:04,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN 2024-12-03T06:22:04,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN 2024-12-03T06:22:04,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ddf2a494d8d0fcd31eac685512f5644, server=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 923ece5c25680df02ff5f922d838bc62, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae2330e378b6ad4b1a3e345529a0b27f, server=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,481 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,481 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,481 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,482 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,482 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,482 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,482 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,482 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,482 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,482 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,482 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,482 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,482 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:04,482 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,482 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN 2024-12-03T06:22:04,482 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,482 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN 2024-12-03T06:22:04,482 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,482 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN 2024-12-03T06:22:04,482 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,482 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN 2024-12-03T06:22:04,482 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,482 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN 2024-12-03T06:22:04,482 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ddf2a494d8d0fcd31eac685512f5644, server=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,482 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,482 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 923ece5c25680df02ff5f922d838bc62, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,482 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae2330e378b6ad4b1a3e345529a0b27f, server=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,482 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-12-03T06:22:04,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=61 2024-12-03T06:22:04,483 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=122cddc95ec8,45577,1733206917466, table=hbase:meta, region=1588230740 2024-12-03T06:22:04,484 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 122cddc95ec8,45577,1733206917466, state=OPEN 2024-12-03T06:22:04,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,494 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,494 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,494 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,494 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,495 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN to state=OFFLINE, location=null, table=test, region=4e8636a0790f504025df15b4adf65735 to restore RIT 2024-12-03T06:22:04,495 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN to state=OFFLINE, location=null, table=test, region=ae2330e378b6ad4b1a3e345529a0b27f to restore RIT 2024-12-03T06:22:04,495 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN to state=OFFLINE, location=null, table=test, region=1ddf2a494d8d0fcd31eac685512f5644 to restore RIT 2024-12-03T06:22:04,495 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN to state=OFFLINE, location=null, table=test, region=663ef4d3dbbcf7b1bb25bba1055da980 to restore RIT 2024-12-03T06:22:04,495 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN to state=OFFLINE, location=null, table=test, region=923ece5c25680df02ff5f922d838bc62 to restore RIT 2024-12-03T06:22:04,495 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN to state=OFFLINE, location=null, table=test, region=07dbe58fce62a9e98b77016cec645f5e to restore RIT 2024-12-03T06:22:04,495 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN to state=OFFLINE, location=null, table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8 to restore RIT 2024-12-03T06:22:04,495 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN to state=OFFLINE, location=null, table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301 to restore RIT 2024-12-03T06:22:04,495 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733206954495 2024-12-03T06:22:04,495 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T06:22:04,496 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,498 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T06:22:04,499 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T06:22:04,499 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 923ece5c25680df02ff5f922d838bc62, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,499 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 923ece5c25680df02ff5f922d838bc62, server=122cddc95ec8,45577,1733206917466 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,501 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663ef4d3dbbcf7b1bb25bba1055da980, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,502 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b4b0967914401ebc70c24d4b56a381, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,502 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bbc9d78d890c8d774a80646d9983ed6, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,502 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=33124cac09cf3a2502ce5e473bf614f4, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,502 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e884340ece2e3eb17a321ed6d089e5e7, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,502 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e48f9dd0d4bf7a7f5e5fb4dac950f301, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,502 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca916e60e459eeefe786a877b8700115, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,503 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d719355720f98b8bc93f39a6130075b, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,503 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a332f3da50b1f1fe35aa4e0307fe35f8, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,503 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c9e860d892b47160f56d24cb24e6010, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,503 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f495460f3e799a3e64378f8879c721bb, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,503 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ddf2a494d8d0fcd31eac685512f5644, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,503 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=050ab97be4cc9f90a155d3b261335a0d, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,503 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5f7ab205e6c897503e3ec729aaa3170, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,504 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6fcc2e40df410122347a4873dfd1129, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,504 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07dbe58fce62a9e98b77016cec645f5e, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,504 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6badf4f9ef3213a92a5f6ace694c6482, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,504 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afa4ab371a599cc21eb81082233626e3, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,504 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8636a0790f504025df15b4adf65735, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,504 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e061778c0efa365f63836d0691ff7a8, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,504 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3e1e4a86d09a1acb1fcf9d92f129898, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,504 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923ece5c25680df02ff5f922d838bc62, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,505 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9f6e863b0ed7a188cb723f7a298ba2c, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,505 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b148e6fa0896eea84e6c38d5293e44a, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,505 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae2330e378b6ad4b1a3e345529a0b27f, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,505 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=671297fbd74543fd2b2deed06d08e6a7, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,505 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T06:22:04,505 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733206984505 2024-12-03T06:22:04,505 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733207044505 2024-12-03T06:22:04,505 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-03T06:22:04,505 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T06:22:04,505 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,505 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T06:22:04,505 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T06:22:04,506 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T06:22:04,506 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T06:22:04,506 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T06:22:04,506 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T06:22:04,507 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T06:22:04,509 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T06:22:04,509 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,509 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,509 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,509 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T06:22:04,511 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T06:22:04,513 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true 2024-12-03T06:22:04,513 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,513 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,513 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T06:22:04,513 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,513 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,513 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,513 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,513 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,513 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:04,513 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN 2024-12-03T06:22:04,513 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN 2024-12-03T06:22:04,513 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN 2024-12-03T06:22:04,513 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN 2024-12-03T06:22:04,513 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN 2024-12-03T06:22:04,513 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ddf2a494d8d0fcd31eac685512f5644, server=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,513 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,513 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 923ece5c25680df02ff5f922d838bc62, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,513 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,513 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,513 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,514 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,514 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,514 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,514 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,514 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,514 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,514 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,514 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,514 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,514 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:04,514 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,514 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN 2024-12-03T06:22:04,514 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,514 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN 2024-12-03T06:22:04,514 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,514 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN 2024-12-03T06:22:04,514 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,514 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN 2024-12-03T06:22:04,514 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,514 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN 2024-12-03T06:22:04,514 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ddf2a494d8d0fcd31eac685512f5644, server=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,514 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,514 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 923ece5c25680df02ff5f922d838bc62, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,514 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-12-03T06:22:04,515 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=61 2024-12-03T06:22:04,515 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=122cddc95ec8,45577,1733206917466, table=hbase:meta, region=1588230740 2024-12-03T06:22:04,516 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 122cddc95ec8,45577,1733206917466, state=OPEN 2024-12-03T06:22:04,517 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,517 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,527 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,527 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,527 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,527 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,527 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,527 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,528 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN to state=OFFLINE, location=null, table=test, region=4e8636a0790f504025df15b4adf65735 to restore RIT 2024-12-03T06:22:04,528 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN to state=OFFLINE, location=null, table=test, region=ae2330e378b6ad4b1a3e345529a0b27f to restore RIT 2024-12-03T06:22:04,528 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN to state=OFFLINE, location=null, table=test, region=1ddf2a494d8d0fcd31eac685512f5644 to restore RIT 2024-12-03T06:22:04,528 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN to state=OFFLINE, location=null, table=test, region=663ef4d3dbbcf7b1bb25bba1055da980 to restore RIT 2024-12-03T06:22:04,528 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN to state=OFFLINE, location=null, table=test, region=923ece5c25680df02ff5f922d838bc62 to restore RIT 2024-12-03T06:22:04,528 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN to state=OFFLINE, location=null, table=test, region=07dbe58fce62a9e98b77016cec645f5e to restore RIT 2024-12-03T06:22:04,529 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN to state=OFFLINE, location=null, table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8 to restore RIT 2024-12-03T06:22:04,529 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN to state=OFFLINE, location=null, table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301 to restore RIT 2024-12-03T06:22:04,529 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733206954529 2024-12-03T06:22:04,529 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T06:22:04,529 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,531 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T06:22:04,533 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T06:22:04,533 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,533 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,536 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663ef4d3dbbcf7b1bb25bba1055da980, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,536 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b4b0967914401ebc70c24d4b56a381, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,536 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bbc9d78d890c8d774a80646d9983ed6, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=33124cac09cf3a2502ce5e473bf614f4, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e884340ece2e3eb17a321ed6d089e5e7, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e48f9dd0d4bf7a7f5e5fb4dac950f301, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca916e60e459eeefe786a877b8700115, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d719355720f98b8bc93f39a6130075b, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a332f3da50b1f1fe35aa4e0307fe35f8, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c9e860d892b47160f56d24cb24e6010, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f495460f3e799a3e64378f8879c721bb, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ddf2a494d8d0fcd31eac685512f5644, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=050ab97be4cc9f90a155d3b261335a0d, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5f7ab205e6c897503e3ec729aaa3170, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6fcc2e40df410122347a4873dfd1129, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07dbe58fce62a9e98b77016cec645f5e, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6badf4f9ef3213a92a5f6ace694c6482, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afa4ab371a599cc21eb81082233626e3, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8636a0790f504025df15b4adf65735, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e061778c0efa365f63836d0691ff7a8, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3e1e4a86d09a1acb1fcf9d92f129898, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923ece5c25680df02ff5f922d838bc62, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9f6e863b0ed7a188cb723f7a298ba2c, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b148e6fa0896eea84e6c38d5293e44a, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae2330e378b6ad4b1a3e345529a0b27f, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=671297fbd74543fd2b2deed06d08e6a7, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,540 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T06:22:04,540 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733206984540 2024-12-03T06:22:04,540 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733207044540 2024-12-03T06:22:04,540 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-12-03T06:22:04,540 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T06:22:04,540 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,540 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T06:22:04,540 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T06:22:04,540 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T06:22:04,540 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T06:22:04,540 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T06:22:04,540 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T06:22:04,541 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T06:22:04,543 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T06:22:04,543 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,543 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,543 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,543 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,544 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,544 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,544 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,544 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,544 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,544 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,544 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,544 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,544 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,544 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,544 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,544 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T06:22:04,545 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T06:22:04,546 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true 2024-12-03T06:22:04,546 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,546 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,546 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T06:22:04,546 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,547 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,547 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,547 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,547 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,547 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:04,547 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN 2024-12-03T06:22:04,547 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN 2024-12-03T06:22:04,547 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN 2024-12-03T06:22:04,547 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN 2024-12-03T06:22:04,547 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN 2024-12-03T06:22:04,547 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ddf2a494d8d0fcd31eac685512f5644, server=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,547 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,547 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,547 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,547 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,547 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,547 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,547 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,547 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,547 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,547 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,547 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,547 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,547 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,547 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:04,547 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,547 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN 2024-12-03T06:22:04,547 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,548 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN 2024-12-03T06:22:04,548 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,548 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN 2024-12-03T06:22:04,548 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,548 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN 2024-12-03T06:22:04,548 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,548 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN 2024-12-03T06:22:04,548 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ddf2a494d8d0fcd31eac685512f5644, server=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,548 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,548 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-03T06:22:04,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=61 2024-12-03T06:22:04,549 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=122cddc95ec8,45577,1733206917466, table=hbase:meta, region=1588230740 2024-12-03T06:22:04,549 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 122cddc95ec8,45577,1733206917466, state=OPEN 2024-12-03T06:22:04,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,560 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,561 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,561 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,561 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,561 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN to state=OFFLINE, location=null, table=test, region=4e8636a0790f504025df15b4adf65735 to restore RIT 2024-12-03T06:22:04,561 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN to state=OFFLINE, location=null, table=test, region=ae2330e378b6ad4b1a3e345529a0b27f to restore RIT 2024-12-03T06:22:04,561 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN to state=OFFLINE, location=null, table=test, region=1ddf2a494d8d0fcd31eac685512f5644 to restore RIT 2024-12-03T06:22:04,561 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN to state=OFFLINE, location=null, table=test, region=663ef4d3dbbcf7b1bb25bba1055da980 to restore RIT 2024-12-03T06:22:04,561 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN to state=OFFLINE, location=null, table=test, region=923ece5c25680df02ff5f922d838bc62 to restore RIT 2024-12-03T06:22:04,561 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN to state=OFFLINE, location=null, table=test, region=07dbe58fce62a9e98b77016cec645f5e to restore RIT 2024-12-03T06:22:04,561 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN to state=OFFLINE, location=null, table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8 to restore RIT 2024-12-03T06:22:04,562 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN to state=OFFLINE, location=null, table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301 to restore RIT 2024-12-03T06:22:04,562 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733206954562 2024-12-03T06:22:04,562 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T06:22:04,562 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,563 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T06:22:04,564 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T06:22:04,564 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ddf2a494d8d0fcd31eac685512f5644, server=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,564 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,565 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ddf2a494d8d0fcd31eac685512f5644, server=122cddc95ec8,43629,1733206917367 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,569 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663ef4d3dbbcf7b1bb25bba1055da980, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,569 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b4b0967914401ebc70c24d4b56a381, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,569 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bbc9d78d890c8d774a80646d9983ed6, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,569 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=33124cac09cf3a2502ce5e473bf614f4, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,570 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e884340ece2e3eb17a321ed6d089e5e7, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,570 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e48f9dd0d4bf7a7f5e5fb4dac950f301, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,570 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca916e60e459eeefe786a877b8700115, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,570 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d719355720f98b8bc93f39a6130075b, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,570 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a332f3da50b1f1fe35aa4e0307fe35f8, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c9e860d892b47160f56d24cb24e6010, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f495460f3e799a3e64378f8879c721bb, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ddf2a494d8d0fcd31eac685512f5644, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=050ab97be4cc9f90a155d3b261335a0d, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5f7ab205e6c897503e3ec729aaa3170, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6fcc2e40df410122347a4873dfd1129, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07dbe58fce62a9e98b77016cec645f5e, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6badf4f9ef3213a92a5f6ace694c6482, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afa4ab371a599cc21eb81082233626e3, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8636a0790f504025df15b4adf65735, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e061778c0efa365f63836d0691ff7a8, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3e1e4a86d09a1acb1fcf9d92f129898, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923ece5c25680df02ff5f922d838bc62, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9f6e863b0ed7a188cb723f7a298ba2c, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b148e6fa0896eea84e6c38d5293e44a, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae2330e378b6ad4b1a3e345529a0b27f, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,574 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=671297fbd74543fd2b2deed06d08e6a7, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,574 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T06:22:04,574 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733206984574 2024-12-03T06:22:04,574 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733207044574 2024-12-03T06:22:04,574 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-12-03T06:22:04,574 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T06:22:04,574 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,574 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T06:22:04,574 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T06:22:04,575 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T06:22:04,575 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T06:22:04,575 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T06:22:04,575 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T06:22:04,576 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T06:22:04,577 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T06:22:04,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,579 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T06:22:04,580 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T06:22:04,582 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true 2024-12-03T06:22:04,582 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,582 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,582 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T06:22:04,582 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,582 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,582 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,582 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,582 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,582 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:04,582 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN 2024-12-03T06:22:04,582 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN 2024-12-03T06:22:04,582 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN 2024-12-03T06:22:04,582 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN 2024-12-03T06:22:04,582 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN 2024-12-03T06:22:04,582 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ddf2a494d8d0fcd31eac685512f5644, server=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,582 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,582 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,582 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,582 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,583 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,583 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,583 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,583 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,583 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,583 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,583 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,583 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,583 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:04,583 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,583 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN 2024-12-03T06:22:04,583 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,583 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN 2024-12-03T06:22:04,583 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,583 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN 2024-12-03T06:22:04,583 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,583 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN 2024-12-03T06:22:04,583 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,583 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN 2024-12-03T06:22:04,583 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ddf2a494d8d0fcd31eac685512f5644, server=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,583 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-12-03T06:22:04,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=61 2024-12-03T06:22:04,584 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=122cddc95ec8,45577,1733206917466, table=hbase:meta, region=1588230740 2024-12-03T06:22:04,584 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 122cddc95ec8,45577,1733206917466, state=OPEN 2024-12-03T06:22:04,594 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,594 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,594 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,594 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,594 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN to state=OFFLINE, location=null, table=test, region=4e8636a0790f504025df15b4adf65735 to restore RIT 2024-12-03T06:22:04,595 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN to state=OFFLINE, location=null, table=test, region=ae2330e378b6ad4b1a3e345529a0b27f to restore RIT 2024-12-03T06:22:04,595 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN to state=OFFLINE, location=null, table=test, region=1ddf2a494d8d0fcd31eac685512f5644 to restore RIT 2024-12-03T06:22:04,595 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN to state=OFFLINE, location=null, table=test, region=663ef4d3dbbcf7b1bb25bba1055da980 to restore RIT 2024-12-03T06:22:04,595 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN to state=OFFLINE, location=null, table=test, region=923ece5c25680df02ff5f922d838bc62 to restore RIT 2024-12-03T06:22:04,595 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN to state=OFFLINE, location=null, table=test, region=07dbe58fce62a9e98b77016cec645f5e to restore RIT 2024-12-03T06:22:04,595 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN to state=OFFLINE, location=null, table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8 to restore RIT 2024-12-03T06:22:04,595 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN to state=OFFLINE, location=null, table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301 to restore RIT 2024-12-03T06:22:04,595 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733206954595 2024-12-03T06:22:04,595 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T06:22:04,595 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,597 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T06:22:04,598 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T06:22:04,598 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN 2024-12-03T06:22:04,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,598 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,602 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663ef4d3dbbcf7b1bb25bba1055da980, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,602 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b4b0967914401ebc70c24d4b56a381, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,602 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bbc9d78d890c8d774a80646d9983ed6, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,602 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=33124cac09cf3a2502ce5e473bf614f4, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e884340ece2e3eb17a321ed6d089e5e7, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e48f9dd0d4bf7a7f5e5fb4dac950f301, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca916e60e459eeefe786a877b8700115, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d719355720f98b8bc93f39a6130075b, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a332f3da50b1f1fe35aa4e0307fe35f8, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c9e860d892b47160f56d24cb24e6010, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f495460f3e799a3e64378f8879c721bb, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ddf2a494d8d0fcd31eac685512f5644, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=050ab97be4cc9f90a155d3b261335a0d, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5f7ab205e6c897503e3ec729aaa3170, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6fcc2e40df410122347a4873dfd1129, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07dbe58fce62a9e98b77016cec645f5e, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6badf4f9ef3213a92a5f6ace694c6482, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afa4ab371a599cc21eb81082233626e3, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8636a0790f504025df15b4adf65735, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e061778c0efa365f63836d0691ff7a8, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3e1e4a86d09a1acb1fcf9d92f129898, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923ece5c25680df02ff5f922d838bc62, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9f6e863b0ed7a188cb723f7a298ba2c, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b148e6fa0896eea84e6c38d5293e44a, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae2330e378b6ad4b1a3e345529a0b27f, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=671297fbd74543fd2b2deed06d08e6a7, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,606 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T06:22:04,607 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733206984606 2024-12-03T06:22:04,607 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733207044607 2024-12-03T06:22:04,607 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-12-03T06:22:04,607 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T06:22:04,607 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,607 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T06:22:04,607 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T06:22:04,607 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T06:22:04,607 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T06:22:04,607 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T06:22:04,608 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T06:22:04,609 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T06:22:04,611 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T06:22:04,611 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,611 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,611 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,611 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,611 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,611 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,611 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,611 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,611 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,611 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,611 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,611 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,612 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,612 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,612 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T06:22:04,613 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T06:22:04,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true 2024-12-03T06:22:04,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T06:22:04,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:04,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN 2024-12-03T06:22:04,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN 2024-12-03T06:22:04,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN 2024-12-03T06:22:04,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN 2024-12-03T06:22:04,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN 2024-12-03T06:22:04,615 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,615 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,616 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,616 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,616 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,616 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,616 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,616 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,616 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,616 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,616 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,616 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,616 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:04,616 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,616 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN 2024-12-03T06:22:04,616 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,616 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN 2024-12-03T06:22:04,616 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,616 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN 2024-12-03T06:22:04,616 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,616 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN 2024-12-03T06:22:04,616 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,616 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN 2024-12-03T06:22:04,617 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-12-03T06:22:04,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=61 2024-12-03T06:22:04,618 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=122cddc95ec8,45577,1733206917466, table=hbase:meta, region=1588230740 2024-12-03T06:22:04,618 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 122cddc95ec8,45577,1733206917466, state=OPEN 2024-12-03T06:22:04,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,627 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,627 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,627 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,627 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,628 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN to state=OFFLINE, location=null, table=test, region=4e8636a0790f504025df15b4adf65735 to restore RIT 2024-12-03T06:22:04,628 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN to state=OFFLINE, location=null, table=test, region=ae2330e378b6ad4b1a3e345529a0b27f to restore RIT 2024-12-03T06:22:04,628 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN to state=OFFLINE, location=null, table=test, region=1ddf2a494d8d0fcd31eac685512f5644 to restore RIT 2024-12-03T06:22:04,628 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN to state=OFFLINE, location=null, table=test, region=663ef4d3dbbcf7b1bb25bba1055da980 to restore RIT 2024-12-03T06:22:04,628 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN to state=OFFLINE, location=null, table=test, region=923ece5c25680df02ff5f922d838bc62 to restore RIT 2024-12-03T06:22:04,628 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN to state=OFFLINE, location=null, table=test, region=07dbe58fce62a9e98b77016cec645f5e to restore RIT 2024-12-03T06:22:04,628 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN to state=OFFLINE, location=null, table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8 to restore RIT 2024-12-03T06:22:04,629 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301, ASSIGN to state=OFFLINE, location=null, table=test, region=e48f9dd0d4bf7a7f5e5fb4dac950f301 to restore RIT 2024-12-03T06:22:04,629 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733206954629 2024-12-03T06:22:04,629 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T06:22:04,629 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,631 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T06:22:04,633 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T06:22:04,633 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN 2024-12-03T06:22:04,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,633 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,635 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663ef4d3dbbcf7b1bb25bba1055da980, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,635 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b4b0967914401ebc70c24d4b56a381, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bbc9d78d890c8d774a80646d9983ed6, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=33124cac09cf3a2502ce5e473bf614f4, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e884340ece2e3eb17a321ed6d089e5e7, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e48f9dd0d4bf7a7f5e5fb4dac950f301, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca916e60e459eeefe786a877b8700115, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d719355720f98b8bc93f39a6130075b, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a332f3da50b1f1fe35aa4e0307fe35f8, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c9e860d892b47160f56d24cb24e6010, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f495460f3e799a3e64378f8879c721bb, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ddf2a494d8d0fcd31eac685512f5644, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=050ab97be4cc9f90a155d3b261335a0d, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5f7ab205e6c897503e3ec729aaa3170, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6fcc2e40df410122347a4873dfd1129, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,638 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07dbe58fce62a9e98b77016cec645f5e, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,638 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6badf4f9ef3213a92a5f6ace694c6482, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,638 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afa4ab371a599cc21eb81082233626e3, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,638 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8636a0790f504025df15b4adf65735, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,638 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e061778c0efa365f63836d0691ff7a8, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,638 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3e1e4a86d09a1acb1fcf9d92f129898, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923ece5c25680df02ff5f922d838bc62, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9f6e863b0ed7a188cb723f7a298ba2c, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b148e6fa0896eea84e6c38d5293e44a, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae2330e378b6ad4b1a3e345529a0b27f, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=671297fbd74543fd2b2deed06d08e6a7, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,639 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T06:22:04,639 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733206984639 2024-12-03T06:22:04,639 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733207044639 2024-12-03T06:22:04,640 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-12-03T06:22:04,640 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T06:22:04,640 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,640 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T06:22:04,640 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T06:22:04,640 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T06:22:04,640 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T06:22:04,640 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T06:22:04,641 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T06:22:04,642 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T06:22:04,644 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T06:22:04,644 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,644 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,644 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,644 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,644 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,644 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,644 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,645 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,645 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,645 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,645 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,645 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,645 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,645 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T06:22:04,646 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T06:22:04,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true 2024-12-03T06:22:04,648 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,648 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,648 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T06:22:04,648 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,648 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,648 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,648 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,648 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,648 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:04,648 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN 2024-12-03T06:22:04,648 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN 2024-12-03T06:22:04,648 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN 2024-12-03T06:22:04,648 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN 2024-12-03T06:22:04,648 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,648 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,648 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,648 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,648 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,648 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,648 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,649 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,649 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,649 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,649 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,649 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,649 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:04,649 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,649 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN 2024-12-03T06:22:04,649 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,649 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN 2024-12-03T06:22:04,649 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,649 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN 2024-12-03T06:22:04,649 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,649 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN 2024-12-03T06:22:04,649 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-12-03T06:22:04,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=61 2024-12-03T06:22:04,651 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=122cddc95ec8,45577,1733206917466, table=hbase:meta, region=1588230740 2024-12-03T06:22:04,651 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 122cddc95ec8,45577,1733206917466, state=OPEN 2024-12-03T06:22:04,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,660 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,660 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,660 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,660 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,661 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN to state=OFFLINE, location=null, table=test, region=4e8636a0790f504025df15b4adf65735 to restore RIT 2024-12-03T06:22:04,661 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN to state=OFFLINE, location=null, table=test, region=ae2330e378b6ad4b1a3e345529a0b27f to restore RIT 2024-12-03T06:22:04,661 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN to state=OFFLINE, location=null, table=test, region=1ddf2a494d8d0fcd31eac685512f5644 to restore RIT 2024-12-03T06:22:04,661 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN to state=OFFLINE, location=null, table=test, region=663ef4d3dbbcf7b1bb25bba1055da980 to restore RIT 2024-12-03T06:22:04,661 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN to state=OFFLINE, location=null, table=test, region=923ece5c25680df02ff5f922d838bc62 to restore RIT 2024-12-03T06:22:04,661 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN to state=OFFLINE, location=null, table=test, region=07dbe58fce62a9e98b77016cec645f5e to restore RIT 2024-12-03T06:22:04,662 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8, ASSIGN to state=OFFLINE, location=null, table=test, region=a332f3da50b1f1fe35aa4e0307fe35f8 to restore RIT 2024-12-03T06:22:04,662 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733206954662 2024-12-03T06:22:04,662 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T06:22:04,662 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,663 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T06:22:04,665 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T06:22:04,665 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN 2024-12-03T06:22:04,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,665 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663ef4d3dbbcf7b1bb25bba1055da980, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b4b0967914401ebc70c24d4b56a381, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bbc9d78d890c8d774a80646d9983ed6, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=33124cac09cf3a2502ce5e473bf614f4, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e884340ece2e3eb17a321ed6d089e5e7, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e48f9dd0d4bf7a7f5e5fb4dac950f301, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca916e60e459eeefe786a877b8700115, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d719355720f98b8bc93f39a6130075b, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a332f3da50b1f1fe35aa4e0307fe35f8, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c9e860d892b47160f56d24cb24e6010, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f495460f3e799a3e64378f8879c721bb, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ddf2a494d8d0fcd31eac685512f5644, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=050ab97be4cc9f90a155d3b261335a0d, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5f7ab205e6c897503e3ec729aaa3170, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6fcc2e40df410122347a4873dfd1129, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07dbe58fce62a9e98b77016cec645f5e, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6badf4f9ef3213a92a5f6ace694c6482, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afa4ab371a599cc21eb81082233626e3, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8636a0790f504025df15b4adf65735, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e061778c0efa365f63836d0691ff7a8, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3e1e4a86d09a1acb1fcf9d92f129898, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923ece5c25680df02ff5f922d838bc62, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9f6e863b0ed7a188cb723f7a298ba2c, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b148e6fa0896eea84e6c38d5293e44a, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae2330e378b6ad4b1a3e345529a0b27f, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=671297fbd74543fd2b2deed06d08e6a7, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,672 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T06:22:04,672 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733206984672 2024-12-03T06:22:04,672 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733207044672 2024-12-03T06:22:04,672 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-03T06:22:04,672 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T06:22:04,672 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,672 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T06:22:04,672 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T06:22:04,672 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T06:22:04,673 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T06:22:04,673 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T06:22:04,673 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T06:22:04,674 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T06:22:04,675 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T06:22:04,675 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,675 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,675 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,676 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,676 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,676 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,676 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,676 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,676 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,676 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,676 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,676 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,676 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T06:22:04,676 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T06:22:04,678 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true 2024-12-03T06:22:04,678 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,678 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,678 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T06:22:04,678 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,678 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,678 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,678 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,678 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,678 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:04,678 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN 2024-12-03T06:22:04,678 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN 2024-12-03T06:22:04,678 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN 2024-12-03T06:22:04,678 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,678 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,678 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,678 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,678 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,678 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,678 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,678 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,678 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,678 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,678 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,678 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,678 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:04,678 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,678 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN 2024-12-03T06:22:04,679 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,679 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN 2024-12-03T06:22:04,679 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,679 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN 2024-12-03T06:22:04,679 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-03T06:22:04,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=61 2024-12-03T06:22:04,680 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=122cddc95ec8,45577,1733206917466, table=hbase:meta, region=1588230740 2024-12-03T06:22:04,680 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 122cddc95ec8,45577,1733206917466, state=OPEN 2024-12-03T06:22:04,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,690 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,690 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,690 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,691 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,691 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN to state=OFFLINE, location=null, table=test, region=4e8636a0790f504025df15b4adf65735 to restore RIT 2024-12-03T06:22:04,691 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN to state=OFFLINE, location=null, table=test, region=ae2330e378b6ad4b1a3e345529a0b27f to restore RIT 2024-12-03T06:22:04,691 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN to state=OFFLINE, location=null, table=test, region=1ddf2a494d8d0fcd31eac685512f5644 to restore RIT 2024-12-03T06:22:04,691 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN to state=OFFLINE, location=null, table=test, region=663ef4d3dbbcf7b1bb25bba1055da980 to restore RIT 2024-12-03T06:22:04,692 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN to state=OFFLINE, location=null, table=test, region=923ece5c25680df02ff5f922d838bc62 to restore RIT 2024-12-03T06:22:04,692 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07dbe58fce62a9e98b77016cec645f5e, ASSIGN to state=OFFLINE, location=null, table=test, region=07dbe58fce62a9e98b77016cec645f5e to restore RIT 2024-12-03T06:22:04,692 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733206954692 2024-12-03T06:22:04,692 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T06:22:04,692 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,694 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T06:22:04,695 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T06:22:04,696 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN 2024-12-03T06:22:04,696 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,696 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663ef4d3dbbcf7b1bb25bba1055da980, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b4b0967914401ebc70c24d4b56a381, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bbc9d78d890c8d774a80646d9983ed6, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=33124cac09cf3a2502ce5e473bf614f4, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e884340ece2e3eb17a321ed6d089e5e7, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e48f9dd0d4bf7a7f5e5fb4dac950f301, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca916e60e459eeefe786a877b8700115, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d719355720f98b8bc93f39a6130075b, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a332f3da50b1f1fe35aa4e0307fe35f8, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c9e860d892b47160f56d24cb24e6010, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f495460f3e799a3e64378f8879c721bb, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ddf2a494d8d0fcd31eac685512f5644, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=050ab97be4cc9f90a155d3b261335a0d, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5f7ab205e6c897503e3ec729aaa3170, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6fcc2e40df410122347a4873dfd1129, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07dbe58fce62a9e98b77016cec645f5e, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6badf4f9ef3213a92a5f6ace694c6482, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afa4ab371a599cc21eb81082233626e3, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,700 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8636a0790f504025df15b4adf65735, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,700 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e061778c0efa365f63836d0691ff7a8, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,700 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3e1e4a86d09a1acb1fcf9d92f129898, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,700 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923ece5c25680df02ff5f922d838bc62, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,700 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9f6e863b0ed7a188cb723f7a298ba2c, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,700 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b148e6fa0896eea84e6c38d5293e44a, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,700 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae2330e378b6ad4b1a3e345529a0b27f, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,700 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=671297fbd74543fd2b2deed06d08e6a7, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,700 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T06:22:04,700 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733206984700 2024-12-03T06:22:04,700 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733207044700 2024-12-03T06:22:04,700 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-03T06:22:04,700 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T06:22:04,700 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,701 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T06:22:04,701 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T06:22:04,701 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T06:22:04,701 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T06:22:04,701 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T06:22:04,701 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T06:22:04,702 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T06:22:04,704 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T06:22:04,704 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,704 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,704 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,704 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,704 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,704 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,704 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,704 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,704 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,704 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,704 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,704 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T06:22:04,705 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T06:22:04,706 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true 2024-12-03T06:22:04,706 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,706 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,706 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T06:22:04,706 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,706 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,707 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,707 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,707 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,707 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:04,707 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN 2024-12-03T06:22:04,707 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN 2024-12-03T06:22:04,707 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,707 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,707 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,707 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,707 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,707 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,707 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,707 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,707 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,707 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,707 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,707 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,707 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:04,707 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,707 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN 2024-12-03T06:22:04,707 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,707 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN 2024-12-03T06:22:04,708 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-03T06:22:04,708 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=61 2024-12-03T06:22:04,709 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=122cddc95ec8,45577,1733206917466, table=hbase:meta, region=1588230740 2024-12-03T06:22:04,709 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 122cddc95ec8,45577,1733206917466, state=OPEN 2024-12-03T06:22:04,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,719 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,719 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,719 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,719 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,719 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN to state=OFFLINE, location=null, table=test, region=4e8636a0790f504025df15b4adf65735 to restore RIT 2024-12-03T06:22:04,719 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN to state=OFFLINE, location=null, table=test, region=ae2330e378b6ad4b1a3e345529a0b27f to restore RIT 2024-12-03T06:22:04,719 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN to state=OFFLINE, location=null, table=test, region=1ddf2a494d8d0fcd31eac685512f5644 to restore RIT 2024-12-03T06:22:04,720 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN to state=OFFLINE, location=null, table=test, region=663ef4d3dbbcf7b1bb25bba1055da980 to restore RIT 2024-12-03T06:22:04,720 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=923ece5c25680df02ff5f922d838bc62, ASSIGN to state=OFFLINE, location=null, table=test, region=923ece5c25680df02ff5f922d838bc62 to restore RIT 2024-12-03T06:22:04,720 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733206954720 2024-12-03T06:22:04,720 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T06:22:04,720 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,721 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T06:22:04,722 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T06:22:04,722 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN 2024-12-03T06:22:04,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,722 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663ef4d3dbbcf7b1bb25bba1055da980, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,727 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b4b0967914401ebc70c24d4b56a381, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,727 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bbc9d78d890c8d774a80646d9983ed6, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,727 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=33124cac09cf3a2502ce5e473bf614f4, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,727 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e884340ece2e3eb17a321ed6d089e5e7, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,727 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e48f9dd0d4bf7a7f5e5fb4dac950f301, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca916e60e459eeefe786a877b8700115, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d719355720f98b8bc93f39a6130075b, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a332f3da50b1f1fe35aa4e0307fe35f8, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c9e860d892b47160f56d24cb24e6010, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f495460f3e799a3e64378f8879c721bb, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ddf2a494d8d0fcd31eac685512f5644, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=050ab97be4cc9f90a155d3b261335a0d, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5f7ab205e6c897503e3ec729aaa3170, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6fcc2e40df410122347a4873dfd1129, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07dbe58fce62a9e98b77016cec645f5e, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6badf4f9ef3213a92a5f6ace694c6482, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afa4ab371a599cc21eb81082233626e3, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8636a0790f504025df15b4adf65735, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e061778c0efa365f63836d0691ff7a8, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3e1e4a86d09a1acb1fcf9d92f129898, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923ece5c25680df02ff5f922d838bc62, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9f6e863b0ed7a188cb723f7a298ba2c, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b148e6fa0896eea84e6c38d5293e44a, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae2330e378b6ad4b1a3e345529a0b27f, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=671297fbd74543fd2b2deed06d08e6a7, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,730 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T06:22:04,730 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733206984730 2024-12-03T06:22:04,730 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733207044730 2024-12-03T06:22:04,730 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-12-03T06:22:04,730 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T06:22:04,730 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,731 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T06:22:04,731 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T06:22:04,731 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T06:22:04,731 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T06:22:04,731 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T06:22:04,731 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T06:22:04,732 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T06:22:04,732 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,733 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,735 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T06:22:04,735 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,735 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,735 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,735 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,735 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,735 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,735 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,735 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,735 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,735 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,735 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T06:22:04,736 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T06:22:04,737 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true 2024-12-03T06:22:04,737 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,737 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,737 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T06:22:04,737 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,737 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,737 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,737 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,737 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,737 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:04,737 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN 2024-12-03T06:22:04,738 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,738 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,738 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,738 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,738 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,738 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,738 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,738 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,738 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,738 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,738 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,738 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,738 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:04,738 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,738 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN 2024-12-03T06:22:04,738 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-03T06:22:04,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=61 2024-12-03T06:22:04,739 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=122cddc95ec8,45577,1733206917466, table=hbase:meta, region=1588230740 2024-12-03T06:22:04,740 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 122cddc95ec8,45577,1733206917466, state=OPEN 2024-12-03T06:22:04,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,749 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,749 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,749 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,749 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,749 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN to state=OFFLINE, location=null, table=test, region=4e8636a0790f504025df15b4adf65735 to restore RIT 2024-12-03T06:22:04,750 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN to state=OFFLINE, location=null, table=test, region=ae2330e378b6ad4b1a3e345529a0b27f to restore RIT 2024-12-03T06:22:04,750 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN to state=OFFLINE, location=null, table=test, region=1ddf2a494d8d0fcd31eac685512f5644 to restore RIT 2024-12-03T06:22:04,750 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=663ef4d3dbbcf7b1bb25bba1055da980, ASSIGN to state=OFFLINE, location=null, table=test, region=663ef4d3dbbcf7b1bb25bba1055da980 to restore RIT 2024-12-03T06:22:04,750 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733206954750 2024-12-03T06:22:04,750 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T06:22:04,750 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,752 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T06:22:04,755 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T06:22:04,756 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:04,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663ef4d3dbbcf7b1bb25bba1055da980, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,756 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b4b0967914401ebc70c24d4b56a381, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bbc9d78d890c8d774a80646d9983ed6, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=33124cac09cf3a2502ce5e473bf614f4, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e884340ece2e3eb17a321ed6d089e5e7, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e48f9dd0d4bf7a7f5e5fb4dac950f301, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca916e60e459eeefe786a877b8700115, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d719355720f98b8bc93f39a6130075b, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a332f3da50b1f1fe35aa4e0307fe35f8, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c9e860d892b47160f56d24cb24e6010, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f495460f3e799a3e64378f8879c721bb, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ddf2a494d8d0fcd31eac685512f5644, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=050ab97be4cc9f90a155d3b261335a0d, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5f7ab205e6c897503e3ec729aaa3170, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6fcc2e40df410122347a4873dfd1129, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07dbe58fce62a9e98b77016cec645f5e, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6badf4f9ef3213a92a5f6ace694c6482, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afa4ab371a599cc21eb81082233626e3, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8636a0790f504025df15b4adf65735, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e061778c0efa365f63836d0691ff7a8, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3e1e4a86d09a1acb1fcf9d92f129898, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923ece5c25680df02ff5f922d838bc62, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9f6e863b0ed7a188cb723f7a298ba2c, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b148e6fa0896eea84e6c38d5293e44a, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae2330e378b6ad4b1a3e345529a0b27f, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=671297fbd74543fd2b2deed06d08e6a7, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,759 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T06:22:04,759 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733206984759 2024-12-03T06:22:04,760 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733207044760 2024-12-03T06:22:04,760 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-03T06:22:04,760 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T06:22:04,760 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,760 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T06:22:04,760 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T06:22:04,760 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T06:22:04,760 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T06:22:04,760 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T06:22:04,761 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T06:22:04,762 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T06:22:04,764 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T06:22:04,764 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,764 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,764 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,764 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,764 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,764 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,764 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,764 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,764 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,764 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T06:22:04,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T06:22:04,766 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true 2024-12-03T06:22:04,766 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,767 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,767 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T06:22:04,767 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,767 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,767 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,767 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,767 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,767 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:04,767 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,767 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,767 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,767 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,767 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,767 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,767 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,767 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,767 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,767 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,767 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,767 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,767 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN 2024-12-03T06:22:04,768 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-03T06:22:04,769 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=61 2024-12-03T06:22:04,769 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=122cddc95ec8,45577,1733206917466, table=hbase:meta, region=1588230740 2024-12-03T06:22:04,769 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 122cddc95ec8,45577,1733206917466, state=OPEN 2024-12-03T06:22:04,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,777 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,777 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,777 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,777 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,778 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN to state=OFFLINE, location=null, table=test, region=4e8636a0790f504025df15b4adf65735 to restore RIT 2024-12-03T06:22:04,778 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN to state=OFFLINE, location=null, table=test, region=ae2330e378b6ad4b1a3e345529a0b27f to restore RIT 2024-12-03T06:22:04,778 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1ddf2a494d8d0fcd31eac685512f5644, ASSIGN to state=OFFLINE, location=null, table=test, region=1ddf2a494d8d0fcd31eac685512f5644 to restore RIT 2024-12-03T06:22:04,778 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733206954778 2024-12-03T06:22:04,778 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T06:22:04,778 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,780 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T06:22:04,781 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T06:22:04,782 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,782 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663ef4d3dbbcf7b1bb25bba1055da980, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b4b0967914401ebc70c24d4b56a381, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bbc9d78d890c8d774a80646d9983ed6, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=33124cac09cf3a2502ce5e473bf614f4, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e884340ece2e3eb17a321ed6d089e5e7, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e48f9dd0d4bf7a7f5e5fb4dac950f301, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca916e60e459eeefe786a877b8700115, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d719355720f98b8bc93f39a6130075b, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a332f3da50b1f1fe35aa4e0307fe35f8, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c9e860d892b47160f56d24cb24e6010, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f495460f3e799a3e64378f8879c721bb, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ddf2a494d8d0fcd31eac685512f5644, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=050ab97be4cc9f90a155d3b261335a0d, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5f7ab205e6c897503e3ec729aaa3170, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6fcc2e40df410122347a4873dfd1129, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07dbe58fce62a9e98b77016cec645f5e, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6badf4f9ef3213a92a5f6ace694c6482, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afa4ab371a599cc21eb81082233626e3, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8636a0790f504025df15b4adf65735, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e061778c0efa365f63836d0691ff7a8, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3e1e4a86d09a1acb1fcf9d92f129898, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923ece5c25680df02ff5f922d838bc62, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9f6e863b0ed7a188cb723f7a298ba2c, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b148e6fa0896eea84e6c38d5293e44a, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae2330e378b6ad4b1a3e345529a0b27f, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=671297fbd74543fd2b2deed06d08e6a7, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,788 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T06:22:04,788 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733206984788 2024-12-03T06:22:04,788 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733207044788 2024-12-03T06:22:04,788 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-03T06:22:04,788 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T06:22:04,788 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,788 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T06:22:04,788 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T06:22:04,788 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T06:22:04,788 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T06:22:04,788 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T06:22:04,789 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T06:22:04,790 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T06:22:04,792 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T06:22:04,792 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,792 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,792 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,792 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,792 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,792 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,792 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,793 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T06:22:04,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T06:22:04,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true 2024-12-03T06:22:04,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T06:22:04,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,795 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,795 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,795 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,795 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,795 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,796 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,796 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,796 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,796 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,796 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,796 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN 2024-12-03T06:22:04,796 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-03T06:22:04,797 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=61 2024-12-03T06:22:04,797 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=122cddc95ec8,45577,1733206917466, table=hbase:meta, region=1588230740 2024-12-03T06:22:04,797 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 122cddc95ec8,45577,1733206917466, state=OPEN 2024-12-03T06:22:04,836 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,836 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,837 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,837 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,837 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,837 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,838 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN to state=OFFLINE, location=null, table=test, region=4e8636a0790f504025df15b4adf65735 to restore RIT 2024-12-03T06:22:04,838 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae2330e378b6ad4b1a3e345529a0b27f, ASSIGN to state=OFFLINE, location=null, table=test, region=ae2330e378b6ad4b1a3e345529a0b27f to restore RIT 2024-12-03T06:22:04,839 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733206954839 2024-12-03T06:22:04,840 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T06:22:04,840 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,841 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T06:22:04,842 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T06:22:04,842 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,842 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663ef4d3dbbcf7b1bb25bba1055da980, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b4b0967914401ebc70c24d4b56a381, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bbc9d78d890c8d774a80646d9983ed6, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=33124cac09cf3a2502ce5e473bf614f4, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e884340ece2e3eb17a321ed6d089e5e7, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e48f9dd0d4bf7a7f5e5fb4dac950f301, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca916e60e459eeefe786a877b8700115, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d719355720f98b8bc93f39a6130075b, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a332f3da50b1f1fe35aa4e0307fe35f8, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c9e860d892b47160f56d24cb24e6010, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f495460f3e799a3e64378f8879c721bb, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ddf2a494d8d0fcd31eac685512f5644, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=050ab97be4cc9f90a155d3b261335a0d, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5f7ab205e6c897503e3ec729aaa3170, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6fcc2e40df410122347a4873dfd1129, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07dbe58fce62a9e98b77016cec645f5e, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6badf4f9ef3213a92a5f6ace694c6482, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afa4ab371a599cc21eb81082233626e3, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8636a0790f504025df15b4adf65735, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e061778c0efa365f63836d0691ff7a8, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3e1e4a86d09a1acb1fcf9d92f129898, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923ece5c25680df02ff5f922d838bc62, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9f6e863b0ed7a188cb723f7a298ba2c, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b148e6fa0896eea84e6c38d5293e44a, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae2330e378b6ad4b1a3e345529a0b27f, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=671297fbd74543fd2b2deed06d08e6a7, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,847 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T06:22:04,847 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733206984847 2024-12-03T06:22:04,847 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733207044847 2024-12-03T06:22:04,847 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-03T06:22:04,847 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T06:22:04,847 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,847 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T06:22:04,847 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T06:22:04,848 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T06:22:04,848 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T06:22:04,848 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T06:22:04,848 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T06:22:04,849 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T06:22:04,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T06:22:04,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,851 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T06:22:04,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T06:22:04,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true 2024-12-03T06:22:04,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T06:22:04,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,853 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,853 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,853 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,853 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,853 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,853 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,853 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,853 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,853 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN 2024-12-03T06:22:04,853 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-03T06:22:04,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=61 2024-12-03T06:22:04,854 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=122cddc95ec8,45577,1733206917466, table=hbase:meta, region=1588230740 2024-12-03T06:22:04,854 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 122cddc95ec8,45577,1733206917466, state=OPEN 2024-12-03T06:22:04,873 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,873 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,873 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,874 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,874 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,874 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,874 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e8636a0790f504025df15b4adf65735, ASSIGN to state=OFFLINE, location=null, table=test, region=4e8636a0790f504025df15b4adf65735 to restore RIT 2024-12-03T06:22:04,875 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733206954875 2024-12-03T06:22:04,875 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T06:22:04,875 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,876 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T06:22:04,878 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T06:22:04,878 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,878 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663ef4d3dbbcf7b1bb25bba1055da980, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b4b0967914401ebc70c24d4b56a381, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bbc9d78d890c8d774a80646d9983ed6, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=33124cac09cf3a2502ce5e473bf614f4, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e884340ece2e3eb17a321ed6d089e5e7, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e48f9dd0d4bf7a7f5e5fb4dac950f301, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca916e60e459eeefe786a877b8700115, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d719355720f98b8bc93f39a6130075b, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a332f3da50b1f1fe35aa4e0307fe35f8, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c9e860d892b47160f56d24cb24e6010, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f495460f3e799a3e64378f8879c721bb, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ddf2a494d8d0fcd31eac685512f5644, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=050ab97be4cc9f90a155d3b261335a0d, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5f7ab205e6c897503e3ec729aaa3170, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6fcc2e40df410122347a4873dfd1129, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07dbe58fce62a9e98b77016cec645f5e, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6badf4f9ef3213a92a5f6ace694c6482, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afa4ab371a599cc21eb81082233626e3, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8636a0790f504025df15b4adf65735, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e061778c0efa365f63836d0691ff7a8, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3e1e4a86d09a1acb1fcf9d92f129898, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923ece5c25680df02ff5f922d838bc62, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9f6e863b0ed7a188cb723f7a298ba2c, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b148e6fa0896eea84e6c38d5293e44a, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae2330e378b6ad4b1a3e345529a0b27f, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=671297fbd74543fd2b2deed06d08e6a7, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,883 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T06:22:04,883 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733206984883 2024-12-03T06:22:04,883 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733207044883 2024-12-03T06:22:04,884 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-03T06:22:04,884 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T06:22:04,884 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,884 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T06:22:04,884 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T06:22:04,884 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T06:22:04,884 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T06:22:04,884 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T06:22:04,884 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T06:22:04,885 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T06:22:04,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T06:22:04,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,887 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T06:22:04,887 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T06:22:04,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true 2024-12-03T06:22:04,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T06:22:04,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,888 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,888 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,888 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,888 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,888 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,888 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,888 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,888 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-03T06:22:04,889 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=61 2024-12-03T06:22:04,889 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=122cddc95ec8,45577,1733206917466, table=hbase:meta, region=1588230740 2024-12-03T06:22:04,889 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 122cddc95ec8,45577,1733206917466, state=OPEN 2024-12-03T06:22:04,898 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,899 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,899 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,899 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,900 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733206954900 2024-12-03T06:22:04,900 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T06:22:04,900 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,901 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T06:22:04,903 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T06:22:04,903 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,903 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663ef4d3dbbcf7b1bb25bba1055da980, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b4b0967914401ebc70c24d4b56a381, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bbc9d78d890c8d774a80646d9983ed6, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=33124cac09cf3a2502ce5e473bf614f4, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e884340ece2e3eb17a321ed6d089e5e7, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e48f9dd0d4bf7a7f5e5fb4dac950f301, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca916e60e459eeefe786a877b8700115, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d719355720f98b8bc93f39a6130075b, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a332f3da50b1f1fe35aa4e0307fe35f8, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c9e860d892b47160f56d24cb24e6010, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f495460f3e799a3e64378f8879c721bb, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ddf2a494d8d0fcd31eac685512f5644, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=050ab97be4cc9f90a155d3b261335a0d, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5f7ab205e6c897503e3ec729aaa3170, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6fcc2e40df410122347a4873dfd1129, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07dbe58fce62a9e98b77016cec645f5e, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6badf4f9ef3213a92a5f6ace694c6482, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afa4ab371a599cc21eb81082233626e3, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8636a0790f504025df15b4adf65735, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e061778c0efa365f63836d0691ff7a8, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3e1e4a86d09a1acb1fcf9d92f129898, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923ece5c25680df02ff5f922d838bc62, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9f6e863b0ed7a188cb723f7a298ba2c, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b148e6fa0896eea84e6c38d5293e44a, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae2330e378b6ad4b1a3e345529a0b27f, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=671297fbd74543fd2b2deed06d08e6a7, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,907 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T06:22:04,907 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733206984907 2024-12-03T06:22:04,907 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733207044907 2024-12-03T06:22:04,907 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-03T06:22:04,907 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T06:22:04,907 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,907 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T06:22:04,907 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T06:22:04,907 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T06:22:04,907 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T06:22:04,907 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T06:22:04,908 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T06:22:04,908 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T06:22:04,910 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T06:22:04,910 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,910 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,910 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,910 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,910 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T06:22:04,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T06:22:04,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true 2024-12-03T06:22:04,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T06:22:04,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 2024-12-03T06:22:04,911 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,911 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,911 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,911 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,911 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,911 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.1733206919100, worker=122cddc95ec8,43629,1733206917367 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,911 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-03T06:22:04,912 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=61 2024-12-03T06:22:04,912 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=122cddc95ec8,45577,1733206917466, table=hbase:meta, region=1588230740 2024-12-03T06:22:04,912 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 122cddc95ec8,45577,1733206917466, state=OPEN 2024-12-03T06:22:04,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,919 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,919 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,919 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,919 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,920 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733206954920 2024-12-03T06:22:04,920 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T06:22:04,920 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,921 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T06:22:04,922 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T06:22:04,922 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,923 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663ef4d3dbbcf7b1bb25bba1055da980, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b4b0967914401ebc70c24d4b56a381, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bbc9d78d890c8d774a80646d9983ed6, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=33124cac09cf3a2502ce5e473bf614f4, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e884340ece2e3eb17a321ed6d089e5e7, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e48f9dd0d4bf7a7f5e5fb4dac950f301, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca916e60e459eeefe786a877b8700115, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d719355720f98b8bc93f39a6130075b, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a332f3da50b1f1fe35aa4e0307fe35f8, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c9e860d892b47160f56d24cb24e6010, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f495460f3e799a3e64378f8879c721bb, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ddf2a494d8d0fcd31eac685512f5644, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=050ab97be4cc9f90a155d3b261335a0d, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5f7ab205e6c897503e3ec729aaa3170, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6fcc2e40df410122347a4873dfd1129, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07dbe58fce62a9e98b77016cec645f5e, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6badf4f9ef3213a92a5f6ace694c6482, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afa4ab371a599cc21eb81082233626e3, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8636a0790f504025df15b4adf65735, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e061778c0efa365f63836d0691ff7a8, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3e1e4a86d09a1acb1fcf9d92f129898, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923ece5c25680df02ff5f922d838bc62, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9f6e863b0ed7a188cb723f7a298ba2c, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,927 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b148e6fa0896eea84e6c38d5293e44a, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,927 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae2330e378b6ad4b1a3e345529a0b27f, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,927 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=671297fbd74543fd2b2deed06d08e6a7, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,927 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T06:22:04,927 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733206984927 2024-12-03T06:22:04,927 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733207044927 2024-12-03T06:22:04,927 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-03T06:22:04,927 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T06:22:04,927 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,927 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T06:22:04,927 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T06:22:04,927 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T06:22:04,927 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T06:22:04,927 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T06:22:04,928 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T06:22:04,928 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T06:22:04,930 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T06:22:04,930 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,930 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,930 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,930 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-03T06:22:04,930 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T06:22:04,930 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T06:22:04,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true 2024-12-03T06:22:04,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T06:22:04,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,931 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,931 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,931 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,931 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,932 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,932 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-03T06:22:04,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=61 2024-12-03T06:22:04,932 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=122cddc95ec8,45577,1733206917466, table=hbase:meta, region=1588230740 2024-12-03T06:22:04,933 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 122cddc95ec8,45577,1733206917466, state=OPEN 2024-12-03T06:22:04,940 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,940 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35025 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,946 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,946 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,946 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,946 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,948 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733206954948 2024-12-03T06:22:04,948 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T06:22:04,948 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,949 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T06:22:04,950 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T06:22:04,950 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T06:22:04,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,951 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663ef4d3dbbcf7b1bb25bba1055da980, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b4b0967914401ebc70c24d4b56a381, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bbc9d78d890c8d774a80646d9983ed6, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=33124cac09cf3a2502ce5e473bf614f4, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e884340ece2e3eb17a321ed6d089e5e7, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e48f9dd0d4bf7a7f5e5fb4dac950f301, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca916e60e459eeefe786a877b8700115, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d719355720f98b8bc93f39a6130075b, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a332f3da50b1f1fe35aa4e0307fe35f8, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c9e860d892b47160f56d24cb24e6010, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f495460f3e799a3e64378f8879c721bb, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ddf2a494d8d0fcd31eac685512f5644, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=050ab97be4cc9f90a155d3b261335a0d, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5f7ab205e6c897503e3ec729aaa3170, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6fcc2e40df410122347a4873dfd1129, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07dbe58fce62a9e98b77016cec645f5e, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6badf4f9ef3213a92a5f6ace694c6482, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afa4ab371a599cc21eb81082233626e3, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8636a0790f504025df15b4adf65735, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e061778c0efa365f63836d0691ff7a8, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3e1e4a86d09a1acb1fcf9d92f129898, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923ece5c25680df02ff5f922d838bc62, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9f6e863b0ed7a188cb723f7a298ba2c, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b148e6fa0896eea84e6c38d5293e44a, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae2330e378b6ad4b1a3e345529a0b27f, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=671297fbd74543fd2b2deed06d08e6a7, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,954 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T06:22:04,954 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733206984954 2024-12-03T06:22:04,954 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733207044954 2024-12-03T06:22:04,954 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-03T06:22:04,954 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T06:22:04,954 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,954 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T06:22:04,954 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T06:22:04,955 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T06:22:04,955 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T06:22:04,955 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T06:22:04,955 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T06:22:04,956 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T06:22:04,957 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T06:22:04,957 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,957 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,957 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-03T06:22:04,957 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T06:22:04,958 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T06:22:04,958 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true 2024-12-03T06:22:04,958 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,958 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,958 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T06:22:04,959 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,959 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,959 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,959 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,959 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-03T06:22:04,959 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=61 2024-12-03T06:22:04,959 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=122cddc95ec8,45577,1733206917466, table=hbase:meta, region=1588230740 2024-12-03T06:22:04,960 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 122cddc95ec8,45577,1733206917466, state=OPEN 2024-12-03T06:22:04,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,969 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,969 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,969 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,969 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,970 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733206954970 2024-12-03T06:22:04,970 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T06:22:04,970 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,971 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T06:22:04,972 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T06:22:04,972 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,972 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,972 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,974 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663ef4d3dbbcf7b1bb25bba1055da980, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,974 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b4b0967914401ebc70c24d4b56a381, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,974 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bbc9d78d890c8d774a80646d9983ed6, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,974 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=33124cac09cf3a2502ce5e473bf614f4, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,974 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e884340ece2e3eb17a321ed6d089e5e7, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,974 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e48f9dd0d4bf7a7f5e5fb4dac950f301, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,974 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca916e60e459eeefe786a877b8700115, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d719355720f98b8bc93f39a6130075b, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a332f3da50b1f1fe35aa4e0307fe35f8, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c9e860d892b47160f56d24cb24e6010, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f495460f3e799a3e64378f8879c721bb, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ddf2a494d8d0fcd31eac685512f5644, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=050ab97be4cc9f90a155d3b261335a0d, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5f7ab205e6c897503e3ec729aaa3170, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6fcc2e40df410122347a4873dfd1129, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07dbe58fce62a9e98b77016cec645f5e, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6badf4f9ef3213a92a5f6ace694c6482, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afa4ab371a599cc21eb81082233626e3, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8636a0790f504025df15b4adf65735, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e061778c0efa365f63836d0691ff7a8, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3e1e4a86d09a1acb1fcf9d92f129898, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923ece5c25680df02ff5f922d838bc62, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9f6e863b0ed7a188cb723f7a298ba2c, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b148e6fa0896eea84e6c38d5293e44a, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,976 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae2330e378b6ad4b1a3e345529a0b27f, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,976 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=671297fbd74543fd2b2deed06d08e6a7, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,976 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T06:22:04,976 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733206984976 2024-12-03T06:22:04,976 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733207044976 2024-12-03T06:22:04,976 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-03T06:22:04,976 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T06:22:04,976 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,976 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T06:22:04,976 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T06:22:04,976 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T06:22:04,976 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T06:22:04,976 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T06:22:04,976 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T06:22:04,977 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T06:22:04,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T06:22:04,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:04,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-03T06:22:04,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T06:22:04,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T06:22:04,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true 2024-12-03T06:22:04,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,979 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:04,979 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,979 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:04,979 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-03T06:22:04,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=61 2024-12-03T06:22:04,980 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=122cddc95ec8,45577,1733206917466, table=hbase:meta, region=1588230740 2024-12-03T06:22:04,980 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 122cddc95ec8,45577,1733206917466, state=OPEN 2024-12-03T06:22:04,990 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:04,990 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,990 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,990 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:04,991 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733206954991 2024-12-03T06:22:04,992 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T06:22:04,992 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,993 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T06:22:04,994 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T06:22:04,994 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:04,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,995 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:04,996 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663ef4d3dbbcf7b1bb25bba1055da980, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b4b0967914401ebc70c24d4b56a381, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bbc9d78d890c8d774a80646d9983ed6, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=33124cac09cf3a2502ce5e473bf614f4, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e884340ece2e3eb17a321ed6d089e5e7, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e48f9dd0d4bf7a7f5e5fb4dac950f301, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca916e60e459eeefe786a877b8700115, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d719355720f98b8bc93f39a6130075b, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a332f3da50b1f1fe35aa4e0307fe35f8, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c9e860d892b47160f56d24cb24e6010, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f495460f3e799a3e64378f8879c721bb, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ddf2a494d8d0fcd31eac685512f5644, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=050ab97be4cc9f90a155d3b261335a0d, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5f7ab205e6c897503e3ec729aaa3170, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6fcc2e40df410122347a4873dfd1129, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07dbe58fce62a9e98b77016cec645f5e, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6badf4f9ef3213a92a5f6ace694c6482, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afa4ab371a599cc21eb81082233626e3, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8636a0790f504025df15b4adf65735, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e061778c0efa365f63836d0691ff7a8, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3e1e4a86d09a1acb1fcf9d92f129898, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923ece5c25680df02ff5f922d838bc62, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9f6e863b0ed7a188cb723f7a298ba2c, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b148e6fa0896eea84e6c38d5293e44a, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:04,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae2330e378b6ad4b1a3e345529a0b27f, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=671297fbd74543fd2b2deed06d08e6a7, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:04,998 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T06:22:04,998 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733206984998 2024-12-03T06:22:04,998 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733207044998 2024-12-03T06:22:04,998 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-03T06:22:04,998 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T06:22:04,998 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:04,998 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T06:22:04,998 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T06:22:04,999 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T06:22:04,999 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T06:22:04,999 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T06:22:04,999 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T06:22:05,000 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T06:22:05,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T06:22:05,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-03T06:22:05,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T06:22:05,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T06:22:05,002 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true 2024-12-03T06:22:05,002 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 2024-12-03T06:22:05,002 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:05,002 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 122cddc95ec8%2C37369%2C1733206917509.meta.1733206919488.meta, worker=122cddc95ec8,45577,1733206917466 didn't hold the lock before restarting, skip acquiring lock. 2024-12-03T06:22:05,002 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-03T06:22:05,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=61 2024-12-03T06:22:05,003 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=122cddc95ec8,45577,1733206917466, table=hbase:meta, region=1588230740 2024-12-03T06:22:05,003 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 122cddc95ec8,45577,1733206917466, state=OPEN 2024-12-03T06:22:05,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:05,010 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:05,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:05,010 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:05,010 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:05,011 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:05,011 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733206955011 2024-12-03T06:22:05,012 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T06:22:05,012 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:05,014 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T06:22:05,015 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-03T06:22:05,015 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true 2024-12-03T06:22:05,015 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:05,015 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:05,017 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663ef4d3dbbcf7b1bb25bba1055da980, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:05,017 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b4b0967914401ebc70c24d4b56a381, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:05,017 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bbc9d78d890c8d774a80646d9983ed6, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:05,017 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=33124cac09cf3a2502ce5e473bf614f4, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:05,017 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e884340ece2e3eb17a321ed6d089e5e7, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:05,017 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e48f9dd0d4bf7a7f5e5fb4dac950f301, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:05,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca916e60e459eeefe786a877b8700115, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:05,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d719355720f98b8bc93f39a6130075b, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:05,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a332f3da50b1f1fe35aa4e0307fe35f8, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:05,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c9e860d892b47160f56d24cb24e6010, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:05,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f495460f3e799a3e64378f8879c721bb, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:05,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ddf2a494d8d0fcd31eac685512f5644, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:05,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=050ab97be4cc9f90a155d3b261335a0d, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:05,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5f7ab205e6c897503e3ec729aaa3170, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:05,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6fcc2e40df410122347a4873dfd1129, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:05,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07dbe58fce62a9e98b77016cec645f5e, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:05,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6badf4f9ef3213a92a5f6ace694c6482, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:05,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afa4ab371a599cc21eb81082233626e3, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:05,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8636a0790f504025df15b4adf65735, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:05,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e061778c0efa365f63836d0691ff7a8, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:05,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3e1e4a86d09a1acb1fcf9d92f129898, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:05,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923ece5c25680df02ff5f922d838bc62, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:05,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9f6e863b0ed7a188cb723f7a298ba2c, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:05,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b148e6fa0896eea84e6c38d5293e44a, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:05,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae2330e378b6ad4b1a3e345529a0b27f, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:05,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=671297fbd74543fd2b2deed06d08e6a7, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:05,019 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T06:22:05,019 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733206985019 2024-12-03T06:22:05,019 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733207045019 2024-12-03T06:22:05,019 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-03T06:22:05,019 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-03T06:22:05,019 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:05,019 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T06:22:05,019 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T06:22:05,019 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-03T06:22:05,019 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-03T06:22:05,019 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-03T06:22:05,020 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T06:22:05,021 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-03T06:22:05,022 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-03T06:22:05,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T06:22:05,023 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-03T06:22:05,023 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true 2024-12-03T06:22:05,023 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-03T06:22:05,023 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-03T06:22:05,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=61 2024-12-03T06:22:05,024 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=122cddc95ec8,45577,1733206917466, table=hbase:meta, region=1588230740 2024-12-03T06:22:05,024 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 122cddc95ec8,45577,1733206917466, state=OPEN 2024-12-03T06:22:05,033 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T06:22:05,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:05,035 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:05,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T06:22:05,036 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:05,036 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:05,036 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T06:22:05,037 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733206955037 2024-12-03T06:22:05,037 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T06:22:05,037 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T06:22:05,039 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-03T06:22:05,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=57, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure e48f9dd0d4bf7a7f5e5fb4dac950f301, server=122cddc95ec8,43629,1733206917367:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 122cddc95ec8,37369,1733206917509, splitWal=true, meta=true exec-time=2.5850 sec 2024-12-03T06:22:05,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663ef4d3dbbcf7b1bb25bba1055da980, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:05,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b4b0967914401ebc70c24d4b56a381, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:05,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bbc9d78d890c8d774a80646d9983ed6, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:05,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=33124cac09cf3a2502ce5e473bf614f4, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:05,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e884340ece2e3eb17a321ed6d089e5e7, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:05,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e48f9dd0d4bf7a7f5e5fb4dac950f301, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:05,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca916e60e459eeefe786a877b8700115, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:05,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d719355720f98b8bc93f39a6130075b, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:05,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a332f3da50b1f1fe35aa4e0307fe35f8, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:05,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c9e860d892b47160f56d24cb24e6010, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:05,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f495460f3e799a3e64378f8879c721bb, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:05,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ddf2a494d8d0fcd31eac685512f5644, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:05,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=050ab97be4cc9f90a155d3b261335a0d, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:05,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5f7ab205e6c897503e3ec729aaa3170, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:05,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6fcc2e40df410122347a4873dfd1129, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:05,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07dbe58fce62a9e98b77016cec645f5e, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:05,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6badf4f9ef3213a92a5f6ace694c6482, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:05,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afa4ab371a599cc21eb81082233626e3, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:05,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8636a0790f504025df15b4adf65735, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:05,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e061778c0efa365f63836d0691ff7a8, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:05,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3e1e4a86d09a1acb1fcf9d92f129898, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:05,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923ece5c25680df02ff5f922d838bc62, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:05,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9f6e863b0ed7a188cb723f7a298ba2c, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:05,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b148e6fa0896eea84e6c38d5293e44a, regionState=OPEN, lastHost=122cddc95ec8,45577,1733206917466, regionLocation=122cddc95ec8,45577,1733206917466, openSeqNum=2 2024-12-03T06:22:05,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae2330e378b6ad4b1a3e345529a0b27f, regionState=OPENING, lastHost=122cddc95ec8,37369,1733206917509, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:05,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=671297fbd74543fd2b2deed06d08e6a7, regionState=OPEN, lastHost=122cddc95ec8,43629,1733206917367, regionLocation=122cddc95ec8,43629,1733206917367, openSeqNum=2 2024-12-03T06:22:05,045 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-03T06:22:05,045 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733206985045 2024-12-03T06:22:05,045 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733207045045 2024-12-03T06:22:05,045 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-03T06:22:05,063 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=347 (was 380), OpenFileDescriptor=618 (was 639), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=301 (was 301), ProcessCount=11 (was 11), AvailableMemoryMB=12058 (was 12093) 2024-12-03T06:22:05,063 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-03T06:22:05,063 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T06:22:05,063 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T06:22:05,063 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:22:05,064 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:22:05,064 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T06:22:05,064 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-03T06:22:05,064 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=120209966, stopped=false 2024-12-03T06:22:05,065 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=122cddc95ec8,35025,1733206916647 2024-12-03T06:22:05,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T06:22:05,074 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T06:22:05,074 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:22:05,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:22:05,074 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:05,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T06:22:05,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:22:05,074 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T06:22:05,074 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T06:22:05,074 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:22:05,074 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '122cddc95ec8,43629,1733206917367' ***** 2024-12-03T06:22:05,074 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T06:22:05,074 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '122cddc95ec8,45577,1733206917466' ***** 2024-12-03T06:22:05,075 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T06:22:05,075 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T06:22:05,075 INFO [RS:1;122cddc95ec8:45577 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T06:22:05,075 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T06:22:05,075 INFO [RS:1;122cddc95ec8:45577 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T06:22:05,075 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T06:22:05,075 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.HRegionServer(3091): Received CLOSE for f495460f3e799a3e64378f8879c721bb 2024-12-03T06:22:05,075 INFO [RS:0;122cddc95ec8:43629 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T06:22:05,075 INFO [RS:0;122cddc95ec8:43629 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T06:22:05,075 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.HRegionServer(3091): Received CLOSE for 33124cac09cf3a2502ce5e473bf614f4 2024-12-03T06:22:05,075 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T06:22:05,075 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T06:22:05,076 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T06:22:05,076 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T06:22:05,076 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.HRegionServer(3091): Received CLOSE for 050ab97be4cc9f90a155d3b261335a0d 2024-12-03T06:22:05,076 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.HRegionServer(3091): Received CLOSE for a6fcc2e40df410122347a4873dfd1129 2024-12-03T06:22:05,076 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.HRegionServer(3091): Received CLOSE for e3e1e4a86d09a1acb1fcf9d92f129898 2024-12-03T06:22:05,076 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.HRegionServer(3091): Received CLOSE for 6b148e6fa0896eea84e6c38d5293e44a 2024-12-03T06:22:05,076 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.HRegionServer(3091): Received CLOSE for e884340ece2e3eb17a321ed6d089e5e7 2024-12-03T06:22:05,076 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.HRegionServer(3091): Received CLOSE for 7d719355720f98b8bc93f39a6130075b 2024-12-03T06:22:05,076 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.HRegionServer(3091): Received CLOSE for afa4ab371a599cc21eb81082233626e3 2024-12-03T06:22:05,076 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.HRegionServer(3091): Received CLOSE for 3bbc9d78d890c8d774a80646d9983ed6 2024-12-03T06:22:05,077 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.HRegionServer(959): stopping server 122cddc95ec8,45577,1733206917466 2024-12-03T06:22:05,077 INFO [RS:1;122cddc95ec8:45577 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T06:22:05,077 INFO [RS:1;122cddc95ec8:45577 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;122cddc95ec8:45577. 2024-12-03T06:22:05,077 DEBUG [RS:1;122cddc95ec8:45577 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T06:22:05,077 DEBUG [RS:1;122cddc95ec8:45577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:22:05,077 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f495460f3e799a3e64378f8879c721bb, disabling compactions & flushes 2024-12-03T06:22:05,077 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1733206919969.f495460f3e799a3e64378f8879c721bb. 2024-12-03T06:22:05,077 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1733206919969.f495460f3e799a3e64378f8879c721bb. 2024-12-03T06:22:05,077 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1733206919969.f495460f3e799a3e64378f8879c721bb. after waiting 0 ms 2024-12-03T06:22:05,077 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T06:22:05,077 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1733206919969.f495460f3e799a3e64378f8879c721bb. 2024-12-03T06:22:05,077 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T06:22:05,077 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T06:22:05,077 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-03T06:22:05,081 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.HRegionServer(1321): Waiting on 10 regions to close 2024-12-03T06:22:05,081 DEBUG [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T06:22:05,081 INFO [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T06:22:05,081 DEBUG [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T06:22:05,081 DEBUG [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T06:22:05,081 DEBUG [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T06:22:05,082 DEBUG [RS:1;122cddc95ec8:45577 {}] regionserver.HRegionServer(1351): Waiting on 050ab97be4cc9f90a155d3b261335a0d, 1588230740, 3bbc9d78d890c8d774a80646d9983ed6, 663ef4d3dbbcf7b1bb25bba1055da980, 6b148e6fa0896eea84e6c38d5293e44a, 7d719355720f98b8bc93f39a6130075b, a6fcc2e40df410122347a4873dfd1129, afa4ab371a599cc21eb81082233626e3, e3e1e4a86d09a1acb1fcf9d92f129898, e884340ece2e3eb17a321ed6d089e5e7, f495460f3e799a3e64378f8879c721bb 2024-12-03T06:22:05,082 INFO [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.64 KB heapSize=5.42 KB 2024-12-03T06:22:05,085 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.HRegionServer(3091): Received CLOSE for ca916e60e459eeefe786a877b8700115 2024-12-03T06:22:05,085 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.HRegionServer(3091): Received CLOSE for 6badf4f9ef3213a92a5f6ace694c6482 2024-12-03T06:22:05,085 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.HRegionServer(3091): Received CLOSE for 40b4b0967914401ebc70c24d4b56a381 2024-12-03T06:22:05,085 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 33124cac09cf3a2502ce5e473bf614f4, disabling compactions & flushes 2024-12-03T06:22:05,085 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.HRegionServer(3091): Received CLOSE for e48f9dd0d4bf7a7f5e5fb4dac950f301 2024-12-03T06:22:05,085 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.HRegionServer(3091): Received CLOSE for 671297fbd74543fd2b2deed06d08e6a7 2024-12-03T06:22:05,085 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1733206919969.33124cac09cf3a2502ce5e473bf614f4. 2024-12-03T06:22:05,085 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.HRegionServer(3091): Received CLOSE for 1e061778c0efa365f63836d0691ff7a8 2024-12-03T06:22:05,085 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.HRegionServer(3091): Received CLOSE for b5f7ab205e6c897503e3ec729aaa3170 2024-12-03T06:22:05,085 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1733206919969.33124cac09cf3a2502ce5e473bf614f4. 2024-12-03T06:22:05,086 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.HRegionServer(3091): Received CLOSE for 3c9e860d892b47160f56d24cb24e6010 2024-12-03T06:22:05,086 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.HRegionServer(3091): Received CLOSE for 4e8636a0790f504025df15b4adf65735 2024-12-03T06:22:05,086 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1733206919969.33124cac09cf3a2502ce5e473bf614f4. after waiting 0 ms 2024-12-03T06:22:05,086 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.HRegionServer(3091): Received CLOSE for e9f6e863b0ed7a188cb723f7a298ba2c 2024-12-03T06:22:05,086 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.HRegionServer(959): stopping server 122cddc95ec8,43629,1733206917367 2024-12-03T06:22:05,086 INFO [RS:0;122cddc95ec8:43629 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T06:22:05,086 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1733206919969.33124cac09cf3a2502ce5e473bf614f4. 2024-12-03T06:22:05,086 INFO [RS:0;122cddc95ec8:43629 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;122cddc95ec8:43629. 2024-12-03T06:22:05,086 DEBUG [RS:0;122cddc95ec8:43629 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T06:22:05,086 DEBUG [RS:0;122cddc95ec8:43629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:22:05,086 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-12-03T06:22:05,086 DEBUG [RS:0;122cddc95ec8:43629 {}] regionserver.HRegionServer(1351): Waiting on 1e061778c0efa365f63836d0691ff7a8, 33124cac09cf3a2502ce5e473bf614f4, 3c9e860d892b47160f56d24cb24e6010, 40b4b0967914401ebc70c24d4b56a381, 4e8636a0790f504025df15b4adf65735, 671297fbd74543fd2b2deed06d08e6a7, 6badf4f9ef3213a92a5f6ace694c6482, ae2330e378b6ad4b1a3e345529a0b27f, b5f7ab205e6c897503e3ec729aaa3170, ca916e60e459eeefe786a877b8700115, e48f9dd0d4bf7a7f5e5fb4dac950f301, e9f6e863b0ed7a188cb723f7a298ba2c 2024-12-03T06:22:05,090 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/f495460f3e799a3e64378f8879c721bb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T06:22:05,093 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1733206919969.f495460f3e799a3e64378f8879c721bb. 2024-12-03T06:22:05,094 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f495460f3e799a3e64378f8879c721bb: Waiting for close lock at 1733206925076Running coprocessor pre-close hooks at 1733206925077 (+1 ms)Disabling compacts and flushes for region at 1733206925077Disabling writes for close at 1733206925077Writing region close event to WAL at 1733206925078 (+1 ms)Running coprocessor post-close hooks at 1733206925091 (+13 ms)Closed at 1733206925093 (+2 ms) 2024-12-03T06:22:05,094 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1733206919969.f495460f3e799a3e64378f8879c721bb. 2024-12-03T06:22:05,094 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 050ab97be4cc9f90a155d3b261335a0d, disabling compactions & flushes 2024-12-03T06:22:05,095 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1733206919969.050ab97be4cc9f90a155d3b261335a0d. 2024-12-03T06:22:05,095 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1733206919969.050ab97be4cc9f90a155d3b261335a0d. 2024-12-03T06:22:05,095 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1733206919969.050ab97be4cc9f90a155d3b261335a0d. after waiting 0 ms 2024-12-03T06:22:05,095 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1733206919969.050ab97be4cc9f90a155d3b261335a0d. 2024-12-03T06:22:05,096 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/33124cac09cf3a2502ce5e473bf614f4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T06:22:05,098 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1733206919969.33124cac09cf3a2502ce5e473bf614f4. 2024-12-03T06:22:05,098 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 33124cac09cf3a2502ce5e473bf614f4: Waiting for close lock at 1733206925085Running coprocessor pre-close hooks at 1733206925085Disabling compacts and flushes for region at 1733206925085Disabling writes for close at 1733206925086 (+1 ms)Writing region close event to WAL at 1733206925087 (+1 ms)Running coprocessor post-close hooks at 1733206925098 (+11 ms)Closed at 1733206925098 2024-12-03T06:22:05,098 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1733206919969.33124cac09cf3a2502ce5e473bf614f4. 2024-12-03T06:22:05,098 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ca916e60e459eeefe786a877b8700115, disabling compactions & flushes 2024-12-03T06:22:05,098 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1733206919969.ca916e60e459eeefe786a877b8700115. 2024-12-03T06:22:05,098 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1733206919969.ca916e60e459eeefe786a877b8700115. 2024-12-03T06:22:05,098 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1733206919969.ca916e60e459eeefe786a877b8700115. after waiting 0 ms 2024-12-03T06:22:05,098 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1733206919969.ca916e60e459eeefe786a877b8700115. 2024-12-03T06:22:05,101 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/050ab97be4cc9f90a155d3b261335a0d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T06:22:05,102 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1733206919969.050ab97be4cc9f90a155d3b261335a0d. 2024-12-03T06:22:05,102 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 050ab97be4cc9f90a155d3b261335a0d: Waiting for close lock at 1733206925094Running coprocessor pre-close hooks at 1733206925094Disabling compacts and flushes for region at 1733206925094Disabling writes for close at 1733206925095 (+1 ms)Writing region close event to WAL at 1733206925095Running coprocessor post-close hooks at 1733206925101 (+6 ms)Closed at 1733206925102 (+1 ms) 2024-12-03T06:22:05,102 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1733206919969.050ab97be4cc9f90a155d3b261335a0d. 2024-12-03T06:22:05,102 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a6fcc2e40df410122347a4873dfd1129, disabling compactions & flushes 2024-12-03T06:22:05,102 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1733206919969.a6fcc2e40df410122347a4873dfd1129. 2024-12-03T06:22:05,102 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1733206919969.a6fcc2e40df410122347a4873dfd1129. 2024-12-03T06:22:05,102 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1733206919969.a6fcc2e40df410122347a4873dfd1129. after waiting 0 ms 2024-12-03T06:22:05,102 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1733206919969.a6fcc2e40df410122347a4873dfd1129. 2024-12-03T06:22:05,103 DEBUG [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/.tmp/info/afd276b31d134f34a737c1645d5fbb48 is 138, key is test,fff,1733206919969.e48f9dd0d4bf7a7f5e5fb4dac950f301./info:regioninfo/1733206923989/Put/seqid=0 2024-12-03T06:22:05,104 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/ca916e60e459eeefe786a877b8700115/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T06:22:05,105 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1733206919969.ca916e60e459eeefe786a877b8700115. 2024-12-03T06:22:05,105 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ca916e60e459eeefe786a877b8700115: Waiting for close lock at 1733206925098Running coprocessor pre-close hooks at 1733206925098Disabling compacts and flushes for region at 1733206925098Disabling writes for close at 1733206925098Writing region close event to WAL at 1733206925099 (+1 ms)Running coprocessor post-close hooks at 1733206925104 (+5 ms)Closed at 1733206925104 2024-12-03T06:22:05,105 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1733206919969.ca916e60e459eeefe786a877b8700115. 2024-12-03T06:22:05,105 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6badf4f9ef3213a92a5f6ace694c6482, disabling compactions & flushes 2024-12-03T06:22:05,105 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1733206919969.6badf4f9ef3213a92a5f6ace694c6482. 2024-12-03T06:22:05,105 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1733206919969.6badf4f9ef3213a92a5f6ace694c6482. 2024-12-03T06:22:05,105 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1733206919969.6badf4f9ef3213a92a5f6ace694c6482. after waiting 0 ms 2024-12-03T06:22:05,105 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1733206919969.6badf4f9ef3213a92a5f6ace694c6482. 2024-12-03T06:22:05,111 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/a6fcc2e40df410122347a4873dfd1129/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T06:22:05,111 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/6badf4f9ef3213a92a5f6ace694c6482/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T06:22:05,111 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1733206919969.6badf4f9ef3213a92a5f6ace694c6482. 2024-12-03T06:22:05,112 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1733206919969.a6fcc2e40df410122347a4873dfd1129. 2024-12-03T06:22:05,112 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a6fcc2e40df410122347a4873dfd1129: Waiting for close lock at 1733206925102Running coprocessor pre-close hooks at 1733206925102Disabling compacts and flushes for region at 1733206925102Disabling writes for close at 1733206925102Writing region close event to WAL at 1733206925103 (+1 ms)Running coprocessor post-close hooks at 1733206925112 (+9 ms)Closed at 1733206925112 2024-12-03T06:22:05,112 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1733206919969.a6fcc2e40df410122347a4873dfd1129. 2024-12-03T06:22:05,112 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-12-03T06:22:05,112 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e3e1e4a86d09a1acb1fcf9d92f129898, disabling compactions & flushes 2024-12-03T06:22:05,112 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1733206919969.e3e1e4a86d09a1acb1fcf9d92f129898. 2024-12-03T06:22:05,112 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1733206919969.e3e1e4a86d09a1acb1fcf9d92f129898. 2024-12-03T06:22:05,112 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1733206919969.e3e1e4a86d09a1acb1fcf9d92f129898. after waiting 0 ms 2024-12-03T06:22:05,112 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1733206919969.e3e1e4a86d09a1acb1fcf9d92f129898. 2024-12-03T06:22:05,112 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6badf4f9ef3213a92a5f6ace694c6482: Waiting for close lock at 1733206925105Running coprocessor pre-close hooks at 1733206925105Disabling compacts and flushes for region at 1733206925105Disabling writes for close at 1733206925105Writing region close event to WAL at 1733206925106 (+1 ms)Running coprocessor post-close hooks at 1733206925111 (+5 ms)Closed at 1733206925111 2024-12-03T06:22:05,112 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1733206919969.6badf4f9ef3213a92a5f6ace694c6482. 2024-12-03T06:22:05,112 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 40b4b0967914401ebc70c24d4b56a381, disabling compactions & flushes 2024-12-03T06:22:05,112 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1733206919969.40b4b0967914401ebc70c24d4b56a381. 2024-12-03T06:22:05,112 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1733206919969.40b4b0967914401ebc70c24d4b56a381. 2024-12-03T06:22:05,112 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1733206919969.40b4b0967914401ebc70c24d4b56a381. after waiting 0 ms 2024-12-03T06:22:05,112 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1733206919969.40b4b0967914401ebc70c24d4b56a381. 2024-12-03T06:22:05,123 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/e3e1e4a86d09a1acb1fcf9d92f129898/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T06:22:05,123 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1733206919969.e3e1e4a86d09a1acb1fcf9d92f129898. 2024-12-03T06:22:05,123 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e3e1e4a86d09a1acb1fcf9d92f129898: Waiting for close lock at 1733206925112Running coprocessor pre-close hooks at 1733206925112Disabling compacts and flushes for region at 1733206925112Disabling writes for close at 1733206925112Writing region close event to WAL at 1733206925117 (+5 ms)Running coprocessor post-close hooks at 1733206925123 (+6 ms)Closed at 1733206925123 2024-12-03T06:22:05,124 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1733206919969.e3e1e4a86d09a1acb1fcf9d92f129898. 2024-12-03T06:22:05,124 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6b148e6fa0896eea84e6c38d5293e44a, disabling compactions & flushes 2024-12-03T06:22:05,124 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1733206919969.6b148e6fa0896eea84e6c38d5293e44a. 2024-12-03T06:22:05,124 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1733206919969.6b148e6fa0896eea84e6c38d5293e44a. 2024-12-03T06:22:05,124 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1733206919969.6b148e6fa0896eea84e6c38d5293e44a. after waiting 0 ms 2024-12-03T06:22:05,124 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1733206919969.6b148e6fa0896eea84e6c38d5293e44a. 2024-12-03T06:22:05,130 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/40b4b0967914401ebc70c24d4b56a381/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T06:22:05,131 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1733206919969.40b4b0967914401ebc70c24d4b56a381. 2024-12-03T06:22:05,131 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 40b4b0967914401ebc70c24d4b56a381: Waiting for close lock at 1733206925112Running coprocessor pre-close hooks at 1733206925112Disabling compacts and flushes for region at 1733206925112Disabling writes for close at 1733206925112Writing region close event to WAL at 1733206925121 (+9 ms)Running coprocessor post-close hooks at 1733206925131 (+10 ms)Closed at 1733206925131 2024-12-03T06:22:05,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741869_1045 (size=8177) 2024-12-03T06:22:05,131 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1733206919969.40b4b0967914401ebc70c24d4b56a381. 2024-12-03T06:22:05,131 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e48f9dd0d4bf7a7f5e5fb4dac950f301, disabling compactions & flushes 2024-12-03T06:22:05,131 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1733206919969.e48f9dd0d4bf7a7f5e5fb4dac950f301. 2024-12-03T06:22:05,131 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1733206919969.e48f9dd0d4bf7a7f5e5fb4dac950f301. 2024-12-03T06:22:05,131 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1733206919969.e48f9dd0d4bf7a7f5e5fb4dac950f301. after waiting 0 ms 2024-12-03T06:22:05,131 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1733206919969.e48f9dd0d4bf7a7f5e5fb4dac950f301. 2024-12-03T06:22:05,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741869_1045 (size=8177) 2024-12-03T06:22:05,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741869_1045 (size=8177) 2024-12-03T06:22:05,135 INFO [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.64 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/.tmp/info/afd276b31d134f34a737c1645d5fbb48 2024-12-03T06:22:05,139 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/6b148e6fa0896eea84e6c38d5293e44a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T06:22:05,139 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/e48f9dd0d4bf7a7f5e5fb4dac950f301/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T06:22:05,140 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1733206919969.e48f9dd0d4bf7a7f5e5fb4dac950f301. 2024-12-03T06:22:05,140 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1733206919969.6b148e6fa0896eea84e6c38d5293e44a. 2024-12-03T06:22:05,140 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e48f9dd0d4bf7a7f5e5fb4dac950f301: Waiting for close lock at 1733206925131Running coprocessor pre-close hooks at 1733206925131Disabling compacts and flushes for region at 1733206925131Disabling writes for close at 1733206925131Writing region close event to WAL at 1733206925132 (+1 ms)Running coprocessor post-close hooks at 1733206925140 (+8 ms)Closed at 1733206925140 2024-12-03T06:22:05,140 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6b148e6fa0896eea84e6c38d5293e44a: Waiting for close lock at 1733206925124Running coprocessor pre-close hooks at 1733206925124Disabling compacts and flushes for region at 1733206925124Disabling writes for close at 1733206925124Writing region close event to WAL at 1733206925129 (+5 ms)Running coprocessor post-close hooks at 1733206925140 (+11 ms)Closed at 1733206925140 2024-12-03T06:22:05,140 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1733206919969.e48f9dd0d4bf7a7f5e5fb4dac950f301. 2024-12-03T06:22:05,140 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1733206919969.6b148e6fa0896eea84e6c38d5293e44a. 2024-12-03T06:22:05,140 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e884340ece2e3eb17a321ed6d089e5e7, disabling compactions & flushes 2024-12-03T06:22:05,140 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 671297fbd74543fd2b2deed06d08e6a7, disabling compactions & flushes 2024-12-03T06:22:05,140 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1733206919969.671297fbd74543fd2b2deed06d08e6a7. 2024-12-03T06:22:05,140 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1733206919969.e884340ece2e3eb17a321ed6d089e5e7. 2024-12-03T06:22:05,140 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1733206919969.671297fbd74543fd2b2deed06d08e6a7. 2024-12-03T06:22:05,140 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1733206919969.e884340ece2e3eb17a321ed6d089e5e7. 2024-12-03T06:22:05,140 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1733206919969.671297fbd74543fd2b2deed06d08e6a7. after waiting 0 ms 2024-12-03T06:22:05,140 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1733206919969.e884340ece2e3eb17a321ed6d089e5e7. after waiting 0 ms 2024-12-03T06:22:05,140 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1733206919969.671297fbd74543fd2b2deed06d08e6a7. 2024-12-03T06:22:05,140 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1733206919969.e884340ece2e3eb17a321ed6d089e5e7. 2024-12-03T06:22:05,145 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/671297fbd74543fd2b2deed06d08e6a7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T06:22:05,145 DEBUG [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/.tmp/info/afd276b31d134f34a737c1645d5fbb48 as hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/info/afd276b31d134f34a737c1645d5fbb48 2024-12-03T06:22:05,146 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1733206919969.671297fbd74543fd2b2deed06d08e6a7. 2024-12-03T06:22:05,146 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 671297fbd74543fd2b2deed06d08e6a7: Waiting for close lock at 1733206925140Running coprocessor pre-close hooks at 1733206925140Disabling compacts and flushes for region at 1733206925140Disabling writes for close at 1733206925140Writing region close event to WAL at 1733206925141 (+1 ms)Running coprocessor post-close hooks at 1733206925146 (+5 ms)Closed at 1733206925146 2024-12-03T06:22:05,146 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1733206919969.671297fbd74543fd2b2deed06d08e6a7. 2024-12-03T06:22:05,146 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1e061778c0efa365f63836d0691ff7a8, disabling compactions & flushes 2024-12-03T06:22:05,146 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1733206919969.1e061778c0efa365f63836d0691ff7a8. 2024-12-03T06:22:05,146 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1733206919969.1e061778c0efa365f63836d0691ff7a8. 2024-12-03T06:22:05,146 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1733206919969.1e061778c0efa365f63836d0691ff7a8. after waiting 0 ms 2024-12-03T06:22:05,147 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1733206919969.1e061778c0efa365f63836d0691ff7a8. 2024-12-03T06:22:05,147 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/e884340ece2e3eb17a321ed6d089e5e7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T06:22:05,148 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1733206919969.e884340ece2e3eb17a321ed6d089e5e7. 2024-12-03T06:22:05,148 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e884340ece2e3eb17a321ed6d089e5e7: Waiting for close lock at 1733206925140Running coprocessor pre-close hooks at 1733206925140Disabling compacts and flushes for region at 1733206925140Disabling writes for close at 1733206925140Writing region close event to WAL at 1733206925141 (+1 ms)Running coprocessor post-close hooks at 1733206925148 (+7 ms)Closed at 1733206925148 2024-12-03T06:22:05,148 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1733206919969.e884340ece2e3eb17a321ed6d089e5e7. 2024-12-03T06:22:05,148 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7d719355720f98b8bc93f39a6130075b, disabling compactions & flushes 2024-12-03T06:22:05,148 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1733206919969.7d719355720f98b8bc93f39a6130075b. 2024-12-03T06:22:05,149 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1733206919969.7d719355720f98b8bc93f39a6130075b. 2024-12-03T06:22:05,149 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1733206919969.7d719355720f98b8bc93f39a6130075b. after waiting 0 ms 2024-12-03T06:22:05,149 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1733206919969.7d719355720f98b8bc93f39a6130075b. 2024-12-03T06:22:05,152 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/1e061778c0efa365f63836d0691ff7a8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T06:22:05,153 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1733206919969.1e061778c0efa365f63836d0691ff7a8. 2024-12-03T06:22:05,153 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1e061778c0efa365f63836d0691ff7a8: Waiting for close lock at 1733206925146Running coprocessor pre-close hooks at 1733206925146Disabling compacts and flushes for region at 1733206925146Disabling writes for close at 1733206925146Writing region close event to WAL at 1733206925147 (+1 ms)Running coprocessor post-close hooks at 1733206925153 (+6 ms)Closed at 1733206925153 2024-12-03T06:22:05,153 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1733206919969.1e061778c0efa365f63836d0691ff7a8. 2024-12-03T06:22:05,153 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b5f7ab205e6c897503e3ec729aaa3170, disabling compactions & flushes 2024-12-03T06:22:05,153 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1733206919969.b5f7ab205e6c897503e3ec729aaa3170. 2024-12-03T06:22:05,153 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1733206919969.b5f7ab205e6c897503e3ec729aaa3170. 2024-12-03T06:22:05,153 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1733206919969.b5f7ab205e6c897503e3ec729aaa3170. after waiting 0 ms 2024-12-03T06:22:05,154 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1733206919969.b5f7ab205e6c897503e3ec729aaa3170. 2024-12-03T06:22:05,155 INFO [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/info/afd276b31d134f34a737c1645d5fbb48, entries=24, sequenceid=71, filesize=8.0 K 2024-12-03T06:22:05,155 INFO [regionserver/122cddc95ec8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T06:22:05,155 INFO [regionserver/122cddc95ec8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T06:22:05,156 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/7d719355720f98b8bc93f39a6130075b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T06:22:05,157 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1733206919969.7d719355720f98b8bc93f39a6130075b. 2024-12-03T06:22:05,157 INFO [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 76ms, sequenceid=71, compaction requested=false 2024-12-03T06:22:05,157 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7d719355720f98b8bc93f39a6130075b: Waiting for close lock at 1733206925148Running coprocessor pre-close hooks at 1733206925148Disabling compacts and flushes for region at 1733206925148Disabling writes for close at 1733206925149 (+1 ms)Writing region close event to WAL at 1733206925150 (+1 ms)Running coprocessor post-close hooks at 1733206925157 (+7 ms)Closed at 1733206925157 2024-12-03T06:22:05,157 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1733206919969.7d719355720f98b8bc93f39a6130075b. 2024-12-03T06:22:05,157 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing afa4ab371a599cc21eb81082233626e3, disabling compactions & flushes 2024-12-03T06:22:05,157 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1733206919969.afa4ab371a599cc21eb81082233626e3. 2024-12-03T06:22:05,157 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1733206919969.afa4ab371a599cc21eb81082233626e3. 2024-12-03T06:22:05,157 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1733206919969.afa4ab371a599cc21eb81082233626e3. after waiting 0 ms 2024-12-03T06:22:05,157 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1733206919969.afa4ab371a599cc21eb81082233626e3. 2024-12-03T06:22:05,160 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/b5f7ab205e6c897503e3ec729aaa3170/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T06:22:05,160 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1733206919969.b5f7ab205e6c897503e3ec729aaa3170. 2024-12-03T06:22:05,160 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b5f7ab205e6c897503e3ec729aaa3170: Waiting for close lock at 1733206925153Running coprocessor pre-close hooks at 1733206925153Disabling compacts and flushes for region at 1733206925153Disabling writes for close at 1733206925153Writing region close event to WAL at 1733206925154 (+1 ms)Running coprocessor post-close hooks at 1733206925160 (+6 ms)Closed at 1733206925160 2024-12-03T06:22:05,161 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1733206919969.b5f7ab205e6c897503e3ec729aaa3170. 2024-12-03T06:22:05,161 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3c9e860d892b47160f56d24cb24e6010, disabling compactions & flushes 2024-12-03T06:22:05,161 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1733206919969.3c9e860d892b47160f56d24cb24e6010. 2024-12-03T06:22:05,161 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1733206919969.3c9e860d892b47160f56d24cb24e6010. 2024-12-03T06:22:05,161 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1733206919969.3c9e860d892b47160f56d24cb24e6010. after waiting 0 ms 2024-12-03T06:22:05,161 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1733206919969.3c9e860d892b47160f56d24cb24e6010. 2024-12-03T06:22:05,163 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/afa4ab371a599cc21eb81082233626e3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T06:22:05,164 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1733206919969.afa4ab371a599cc21eb81082233626e3. 2024-12-03T06:22:05,164 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for afa4ab371a599cc21eb81082233626e3: Waiting for close lock at 1733206925157Running coprocessor pre-close hooks at 1733206925157Disabling compacts and flushes for region at 1733206925157Disabling writes for close at 1733206925157Writing region close event to WAL at 1733206925158 (+1 ms)Running coprocessor post-close hooks at 1733206925164 (+6 ms)Closed at 1733206925164 2024-12-03T06:22:05,164 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1733206919969.afa4ab371a599cc21eb81082233626e3. 2024-12-03T06:22:05,164 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3bbc9d78d890c8d774a80646d9983ed6, disabling compactions & flushes 2024-12-03T06:22:05,165 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1733206919969.3bbc9d78d890c8d774a80646d9983ed6. 2024-12-03T06:22:05,165 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1733206919969.3bbc9d78d890c8d774a80646d9983ed6. 2024-12-03T06:22:05,165 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1733206919969.3bbc9d78d890c8d774a80646d9983ed6. after waiting 0 ms 2024-12-03T06:22:05,165 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1733206919969.3bbc9d78d890c8d774a80646d9983ed6. 2024-12-03T06:22:05,167 DEBUG [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/hbase/meta/1588230740/recovered.edits/74.seqid, newMaxSeqId=74, maxSeqId=60 2024-12-03T06:22:05,167 DEBUG [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T06:22:05,167 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/3c9e860d892b47160f56d24cb24e6010/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T06:22:05,168 INFO [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T06:22:05,168 DEBUG [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733206925081Running coprocessor pre-close hooks at 1733206925081Disabling compacts and flushes for region at 1733206925081Disabling writes for close at 1733206925081Obtaining lock to block concurrent updates at 1733206925082 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733206925082Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2708, getHeapSize=5488, getOffHeapSize=0, getCellsCount=24 at 1733206925089 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733206925091 (+2 ms)Flushing 1588230740/info: creating writer at 1733206925091Flushing 1588230740/info: appending metadata at 1733206925102 (+11 ms)Flushing 1588230740/info: closing flushed file at 1733206925102Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b9afeaf: reopening flushed file at 1733206925144 (+42 ms)Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 76ms, sequenceid=71, compaction requested=false at 1733206925157 (+13 ms)Writing region close event to WAL at 1733206925161 (+4 ms)Running coprocessor post-close hooks at 1733206925167 (+6 ms)Closed at 1733206925168 (+1 ms) 2024-12-03T06:22:05,168 DEBUG [RS_CLOSE_META-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-03T06:22:05,168 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1733206919969.3c9e860d892b47160f56d24cb24e6010. 2024-12-03T06:22:05,168 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3c9e860d892b47160f56d24cb24e6010: Waiting for close lock at 1733206925161Running coprocessor pre-close hooks at 1733206925161Disabling compacts and flushes for region at 1733206925161Disabling writes for close at 1733206925161Writing region close event to WAL at 1733206925161Running coprocessor post-close hooks at 1733206925168 (+7 ms)Closed at 1733206925168 2024-12-03T06:22:05,168 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1733206919969.3c9e860d892b47160f56d24cb24e6010. 2024-12-03T06:22:05,168 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4e8636a0790f504025df15b4adf65735, disabling compactions & flushes 2024-12-03T06:22:05,168 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1733206919969.4e8636a0790f504025df15b4adf65735. 2024-12-03T06:22:05,168 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1733206919969.4e8636a0790f504025df15b4adf65735. 2024-12-03T06:22:05,168 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1733206919969.4e8636a0790f504025df15b4adf65735. after waiting 0 ms 2024-12-03T06:22:05,169 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1733206919969.4e8636a0790f504025df15b4adf65735. 2024-12-03T06:22:05,170 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/3bbc9d78d890c8d774a80646d9983ed6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T06:22:05,170 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1733206919969.3bbc9d78d890c8d774a80646d9983ed6. 2024-12-03T06:22:05,170 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3bbc9d78d890c8d774a80646d9983ed6: Waiting for close lock at 1733206925164Running coprocessor pre-close hooks at 1733206925164Disabling compacts and flushes for region at 1733206925164Disabling writes for close at 1733206925165 (+1 ms)Writing region close event to WAL at 1733206925165Running coprocessor post-close hooks at 1733206925170 (+5 ms)Closed at 1733206925170 2024-12-03T06:22:05,170 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1733206919969.3bbc9d78d890c8d774a80646d9983ed6. 2024-12-03T06:22:05,172 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/4e8636a0790f504025df15b4adf65735/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T06:22:05,173 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1733206919969.4e8636a0790f504025df15b4adf65735. 2024-12-03T06:22:05,173 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4e8636a0790f504025df15b4adf65735: Waiting for close lock at 1733206925168Running coprocessor pre-close hooks at 1733206925168Disabling compacts and flushes for region at 1733206925168Disabling writes for close at 1733206925168Writing region close event to WAL at 1733206925169 (+1 ms)Running coprocessor post-close hooks at 1733206925173 (+4 ms)Closed at 1733206925173 2024-12-03T06:22:05,173 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1733206919969.4e8636a0790f504025df15b4adf65735. 2024-12-03T06:22:05,173 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e9f6e863b0ed7a188cb723f7a298ba2c, disabling compactions & flushes 2024-12-03T06:22:05,173 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1733206919969.e9f6e863b0ed7a188cb723f7a298ba2c. 2024-12-03T06:22:05,173 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1733206919969.e9f6e863b0ed7a188cb723f7a298ba2c. 2024-12-03T06:22:05,174 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1733206919969.e9f6e863b0ed7a188cb723f7a298ba2c. after waiting 0 ms 2024-12-03T06:22:05,174 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1733206919969.e9f6e863b0ed7a188cb723f7a298ba2c. 2024-12-03T06:22:05,178 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/e9f6e863b0ed7a188cb723f7a298ba2c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T06:22:05,178 INFO [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1733206919969.e9f6e863b0ed7a188cb723f7a298ba2c. 2024-12-03T06:22:05,178 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e9f6e863b0ed7a188cb723f7a298ba2c: Waiting for close lock at 1733206925173Running coprocessor pre-close hooks at 1733206925173Disabling compacts and flushes for region at 1733206925173Disabling writes for close at 1733206925174 (+1 ms)Writing region close event to WAL at 1733206925174Running coprocessor post-close hooks at 1733206925178 (+4 ms)Closed at 1733206925178 2024-12-03T06:22:05,178 DEBUG [RS_CLOSE_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1733206919969.e9f6e863b0ed7a188cb723f7a298ba2c. 2024-12-03T06:22:05,202 WARN [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,yyy,1733206919969.ae2330e378b6ad4b1a3e345529a0b27f., aborting... java.io.IOException: Failed to report opened region to master: test,yyy,1733206919969.ae2330e378b6ad4b1a3e345529a0b27f. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:22:05,203 ERROR [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2440): ***** ABORTING region server 122cddc95ec8,43629,1733206917367: Failed to open region test,yyy,1733206919969.ae2330e378b6ad4b1a3e345529a0b27f. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,yyy,1733206919969.ae2330e378b6ad4b1a3e345529a0b27f. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:22:05,203 ERROR [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-03T06:22:05,203 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-03T06:22:05,204 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-03T06:22:05,206 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-03T06:22:05,207 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-03T06:22:05,215 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 472612056 }, "NonHeapMemoryUsage": { "committed": 162725888, "init": 7667712, "max": -1, "used": 160204280 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "122cddc95ec8", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 40644, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 132, "ProcessCallTime_min": 0, "ProcessCallTime_max": 28, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 7, "ProcessCallTime_median": 14, "ProcessCallTime_75th_percentile": 21, "ProcessCallTime_90th_percentile": 25, "ProcessCallTime_95th_percentile": 26, "ProcessCallTime_98th_percentile": 27, "ProcessCallTime_99th_percentile": 27, "ProcessCallTime_99.9th_percentile": 27, "ProcessCallTime_TimeRangeCount_0-1": 132, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 132, "QueueCallTime_min": 0, "QueueCallTime_max": 21, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 5, "QueueCallTime_median": 10, "QueueCallTime_75th_percentile": 15, "QueueCallTime_90th_percentile": 18, "QueueCallTime_95th_percentile": 19, "QueueCallTime_98th_percentile": 20, "QueueCallTime_99th_percentile": 20, "QueueCallTime_99.9th_percentile": 20, "QueueCallTime_TimeRangeCount_0-1": 132, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 132, "TotalCallTime_min": 0, "TotalCallTime_max": 28, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 7, "TotalCallTime_median": 14, "TotalCallTime_75th_percentile": 21, "TotalCallTime_90th_percentile": 25, "TotalCallTime_95th_percentile": 26, "TotalCallTime_98th_percentile": 27, "TotalCallTime_99th_percentile": 27, "TotalCallTime_99.9th_percentile": 27, "TotalCallTime_TimeRangeCount_0-1": 132, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 132, "ResponseSize_min": 0, "ResponseSize_max": 22851, "ResponseSize_mean": 4499, "ResponseSize_25th_percentile": 5712, "ResponseSize_median": 11425, "ResponseSize_75th_percentile": 17138, "ResponseSize_90th_percentile": 20565, "ResponseSize_95th_percentile": 21708, "ResponseSize_98th_percentile": 22393, "ResponseSize_99th_percentile": 22622, "ResponseSize_99.9th_percentile": 22828, "ResponseSize_SizeRangeCount_0-10": 132, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 132, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 279, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 132, "sentBytes": 459927 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "122cddc95ec8", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:63630", "tag.serverName": "122cddc95ec8,37369,1733206917509", "tag.clusterId": "c0c7c2db-4b33-4027-9625-bcb0de82bf43", "tag.Context": "regionserver", "tag.Hostname": "122cddc95ec8", "regionCount": 9, "storeCount": 12, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 28672, "memStoreHeapSize": 51520, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733206917509, "averageRegionSize": 3185, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3332, "localBytesRead": 3332, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 18.8, "writeRequestRatePerSecond": 16.400000000000002, "ByteBuffAllocatorHeapAllocationBytes": 36711, "ByteBuffAllocatorPoolAllocationBytes": 465920, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 74, "totalRowActionRequestCount": 176, "readRequestCount": 94, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 82, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 8, "rpcScanRequestCount": 15, "rpcMultiRequestCount": 1, "rpcMutateRequestCount": 56, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 15, "ScanTime_min": 0, "ScanTime_max": 14, "ScanTime_mean": 2, "ScanTime_25th_percentile": 3, "ScanTime_median": 7, "ScanTime_75th_percentile": 10, "ScanTime_90th_percentile": 12, "ScanTime_95th_percentile": 13, "ScanTime_98th_percentile": 13, "ScanTime_99th_percentile": 13, "ScanTime_99.9th_percentile": 13, "ScanTime_TimeRangeCount_0-1": 15, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 56, "Put_min": 1, "Put_max": 7, "Put_mean": 1, "Put_25th_percentile": 2, "Put_median": 4, "Put_75th_percentile": 5, "Put_90th_percentile": 6, "Put_95th_percentile": 6, "Put_98th_percentile": 6, "Put_99th_percentile": 6, "Put_99.9th_percentile": 6, "Put_TimeRangeCount_0-1": 56, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 1, "PutBatch_min": 9, "PutBatch_max": 9, "PutBatch_mean": 9, "PutBatch_25th_percentile": 9, "PutBatch_median": 9, "PutBatch_75th_percentile": 9, "PutBatch_90th_percentile": 9, "PutBatch_95th_percentile": 9, "PutBatch_98th_percentile": 9, "PutBatch_99th_percentile": 9, "PutBatch_99.9th_percentile": 9, "PutBatch_TimeRangeCount_0-1": 1, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 1, "Get_max": 3, "Get_mean": 2, "Get_25th_percentile": 1, "Get_median": 2, "Get_75th_percentile": 2, "Get_90th_percentile": 2, "Get_95th_percentile": 2, "Get_98th_percentile": 2, "Get_99th_percentile": 2, "Get_99.9th_percentile": 2, "Get_TimeRangeCount_0-1": 2, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 15, "ScanSize_min": 0, "ScanSize_max": 22232, "ScanSize_mean": 4465, "ScanSize_25th_percentile": 5558, "ScanSize_median": 11116, "ScanSize_75th_percentile": 16674, "ScanSize_90th_percentile": 20008, "ScanSize_95th_percentile": 21120, "ScanSize_98th_percentile": 21787, "ScanSize_99th_percentile": 22009, "ScanSize_99.9th_percentile": 22209, "ScanSize_SizeRangeCount_0-10": 15, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-03T06:22:05,217 WARN [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=122cddc95ec8:35025 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.handleException(AssignRegionHandler.java:173) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:106) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=122cddc95ec8:35025 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more 2024-12-03T06:22:05,218 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(132): Open test,lll,1733206919969.1ddf2a494d8d0fcd31eac685512f5644. 2024-12-03T06:22:05,218 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7752): Opening region: {ENCODED => 1ddf2a494d8d0fcd31eac685512f5644, NAME => 'test,lll,1733206919969.1ddf2a494d8d0fcd31eac685512f5644.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-12-03T06:22:05,218 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1ddf2a494d8d0fcd31eac685512f5644 2024-12-03T06:22:05,218 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(898): Instantiated test,lll,1733206919969.1ddf2a494d8d0fcd31eac685512f5644.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:05,219 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7794): checking encryption for 1ddf2a494d8d0fcd31eac685512f5644 2024-12-03T06:22:05,219 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7797): checking classloading for 1ddf2a494d8d0fcd31eac685512f5644 2024-12-03T06:22:05,220 INFO [StoreOpener-1ddf2a494d8d0fcd31eac685512f5644-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1ddf2a494d8d0fcd31eac685512f5644 2024-12-03T06:22:05,221 INFO [StoreOpener-1ddf2a494d8d0fcd31eac685512f5644-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ddf2a494d8d0fcd31eac685512f5644 columnFamilyName family 2024-12-03T06:22:05,221 DEBUG [StoreOpener-1ddf2a494d8d0fcd31eac685512f5644-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:05,222 INFO [StoreOpener-1ddf2a494d8d0fcd31eac685512f5644-1 {}] regionserver.HStore(327): Store=1ddf2a494d8d0fcd31eac685512f5644/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:05,222 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1038): replaying wal for 1ddf2a494d8d0fcd31eac685512f5644 2024-12-03T06:22:05,223 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/1ddf2a494d8d0fcd31eac685512f5644 2024-12-03T06:22:05,225 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/data/default/test/1ddf2a494d8d0fcd31eac685512f5644 2024-12-03T06:22:05,225 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1048): stopping wal replay for 1ddf2a494d8d0fcd31eac685512f5644 2024-12-03T06:22:05,225 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1060): Cleaning up temporary data for 1ddf2a494d8d0fcd31eac685512f5644 2024-12-03T06:22:05,227 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): writing seq id for 1ddf2a494d8d0fcd31eac685512f5644 2024-12-03T06:22:05,228 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1114): Opened 1ddf2a494d8d0fcd31eac685512f5644; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69011214, jitterRate=0.02834722399711609}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T06:22:05,228 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1ddf2a494d8d0fcd31eac685512f5644 2024-12-03T06:22:05,228 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1006): Region open journal for 1ddf2a494d8d0fcd31eac685512f5644: Running coprocessor pre-open hook at 1733206925219Writing region info on filesystem at 1733206925219Initializing all the Stores at 1733206925220 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206925220Cleaning up temporary data from old regions at 1733206925225 (+5 ms)Running coprocessor post-open hooks at 1733206925228 (+3 ms)Region opened successfully at 1733206925228 2024-12-03T06:22:05,230 WARN [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,lll,1733206919969.1ddf2a494d8d0fcd31eac685512f5644., aborting... org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 122cddc95ec8,43629,1733206917367 aborting at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1463) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:22:05,230 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: Failed to open region test,lll,1733206919969.1ddf2a494d8d0fcd31eac685512f5644. and can not recover 2024-12-03T06:22:05,270 WARN [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,,1733206919969.663ef4d3dbbcf7b1bb25bba1055da980., aborting... java.io.IOException: Failed to report opened region to master: test,,1733206919969.663ef4d3dbbcf7b1bb25bba1055da980. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:22:05,270 ERROR [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2440): ***** ABORTING region server 122cddc95ec8,45577,1733206917466: Failed to open region test,,1733206919969.663ef4d3dbbcf7b1bb25bba1055da980. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,,1733206919969.663ef4d3dbbcf7b1bb25bba1055da980. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:22:05,271 ERROR [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-03T06:22:05,271 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-03T06:22:05,272 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-03T06:22:05,274 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-03T06:22:05,275 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-03T06:22:05,281 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 474709208 }, "NonHeapMemoryUsage": { "committed": 162988032, "init": 7667712, "max": -1, "used": 160444064 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "122cddc95ec8", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 40644, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 132, "ProcessCallTime_min": 0, "ProcessCallTime_max": 28, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 7, "ProcessCallTime_median": 14, "ProcessCallTime_75th_percentile": 21, "ProcessCallTime_90th_percentile": 25, "ProcessCallTime_95th_percentile": 26, "ProcessCallTime_98th_percentile": 27, "ProcessCallTime_99th_percentile": 27, "ProcessCallTime_99.9th_percentile": 27, "ProcessCallTime_TimeRangeCount_0-1": 132, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 132, "QueueCallTime_min": 0, "QueueCallTime_max": 21, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 5, "QueueCallTime_median": 10, "QueueCallTime_75th_percentile": 15, "QueueCallTime_90th_percentile": 18, "QueueCallTime_95th_percentile": 19, "QueueCallTime_98th_percentile": 20, "QueueCallTime_99th_percentile": 20, "QueueCallTime_99.9th_percentile": 20, "QueueCallTime_TimeRangeCount_0-1": 132, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 132, "TotalCallTime_min": 0, "TotalCallTime_max": 28, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 7, "TotalCallTime_median": 14, "TotalCallTime_75th_percentile": 21, "TotalCallTime_90th_percentile": 25, "TotalCallTime_95th_percentile": 26, "TotalCallTime_98th_percentile": 27, "TotalCallTime_99th_percentile": 27, "TotalCallTime_99.9th_percentile": 27, "TotalCallTime_TimeRangeCount_0-1": 132, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 132, "ResponseSize_min": 0, "ResponseSize_max": 22851, "ResponseSize_mean": 4499, "ResponseSize_25th_percentile": 5712, "ResponseSize_median": 11425, "ResponseSize_75th_percentile": 17138, "ResponseSize_90th_percentile": 20565, "ResponseSize_95th_percentile": 21708, "ResponseSize_98th_percentile": 22393, "ResponseSize_99th_percentile": 22622, "ResponseSize_99.9th_percentile": 22828, "ResponseSize_SizeRangeCount_0-10": 132, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 132, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 279, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 132, "sentBytes": 459927 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "122cddc95ec8", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:63630", "tag.serverName": "122cddc95ec8,37369,1733206917509", "tag.clusterId": "c0c7c2db-4b33-4027-9625-bcb0de82bf43", "tag.Context": "regionserver", "tag.Hostname": "122cddc95ec8", "regionCount": 9, "storeCount": 12, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 28672, "memStoreHeapSize": 51520, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733206917509, "averageRegionSize": 3185, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3332, "localBytesRead": 3332, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 18.8, "writeRequestRatePerSecond": 16.400000000000002, "ByteBuffAllocatorHeapAllocationBytes": 36711, "ByteBuffAllocatorPoolAllocationBytes": 465920, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 74, "totalRowActionRequestCount": 176, "readRequestCount": 94, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 82, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 8, "rpcScanRequestCount": 15, "rpcMultiRequestCount": 1, "rpcMutateRequestCount": 56, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 15, "ScanTime_min": 0, "ScanTime_max": 14, "ScanTime_mean": 2, "ScanTime_25th_percentile": 3, "ScanTime_median": 7, "ScanTime_75th_percentile": 10, "ScanTime_90th_percentile": 12, "ScanTime_95th_percentile": 13, "ScanTime_98th_percentile": 13, "ScanTime_99th_percentile": 13, "ScanTime_99.9th_percentile": 13, "ScanTime_TimeRangeCount_0-1": 15, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 56, "Put_min": 1, "Put_max": 7, "Put_mean": 1, "Put_25th_percentile": 2, "Put_median": 4, "Put_75th_percentile": 5, "Put_90th_percentile": 6, "Put_95th_percentile": 6, "Put_98th_percentile": 6, "Put_99th_percentile": 6, "Put_99.9th_percentile": 6, "Put_TimeRangeCount_0-1": 56, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 1, "PutBatch_min": 9, "PutBatch_max": 9, "PutBatch_mean": 9, "PutBatch_25th_percentile": 9, "PutBatch_median": 9, "PutBatch_75th_percentile": 9, "PutBatch_90th_percentile": 9, "PutBatch_95th_percentile": 9, "PutBatch_98th_percentile": 9, "PutBatch_99th_percentile": 9, "PutBatch_99.9th_percentile": 9, "PutBatch_TimeRangeCount_0-1": 1, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 1, "Get_max": 3, "Get_mean": 2, "Get_25th_percentile": 1, "Get_median": 2, "Get_75th_percentile": 2, "Get_90th_percentile": 2, "Get_95th_percentile": 2, "Get_98th_percentile": 2, "Get_99th_percentile": 2, "Get_99.9th_percentile": 2, "Get_TimeRangeCount_0-1": 2, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 15, "ScanSize_min": 0, "ScanSize_max": 22232, "ScanSize_mean": 4465, "ScanSize_25th_percentile": 5558, "ScanSize_median": 11116, "ScanSize_75th_percentile": 16674, "ScanSize_90th_percentile": 20008, "ScanSize_95th_percentile": 21120, "ScanSize_98th_percentile": 21787, "ScanSize_99th_percentile": 22009, "ScanSize_99.9th_percentile": 22209, "ScanSize_SizeRangeCount_0-10": 15, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-03T06:22:05,282 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.HRegionServer(976): stopping server 122cddc95ec8,45577,1733206917466; all regions closed. 2024-12-03T06:22:05,282 WARN [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=122cddc95ec8:35025 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.handleException(AssignRegionHandler.java:173) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:106) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=122cddc95ec8:35025 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more 2024-12-03T06:22:05,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741865_1041 (size=4192) 2024-12-03T06:22:05,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741865_1041 (size=4192) 2024-12-03T06:22:05,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741865_1041 (size=4192) 2024-12-03T06:22:05,286 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open test,vvv,1733206919969.923ece5c25680df02ff5f922d838bc62. 2024-12-03T06:22:05,286 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => 923ece5c25680df02ff5f922d838bc62, NAME => 'test,vvv,1733206919969.923ece5c25680df02ff5f922d838bc62.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-12-03T06:22:05,286 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.HRegionServer(976): stopping server 122cddc95ec8,43629,1733206917367; all regions closed. 2024-12-03T06:22:05,287 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 923ece5c25680df02ff5f922d838bc62 2024-12-03T06:22:05,287 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated test,vvv,1733206919969.923ece5c25680df02ff5f922d838bc62.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T06:22:05,287 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for 923ece5c25680df02ff5f922d838bc62 2024-12-03T06:22:05,287 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for 923ece5c25680df02ff5f922d838bc62 2024-12-03T06:22:05,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741835_1011 (size=5189) 2024-12-03T06:22:05,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741835_1011 (size=5189) 2024-12-03T06:22:05,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741835_1011 (size=5189) 2024-12-03T06:22:05,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741833_1009 (size=6536) 2024-12-03T06:22:05,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741833_1009 (size=6536) 2024-12-03T06:22:05,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741833_1009 (size=6536) 2024-12-03T06:22:05,290 INFO [StoreOpener-923ece5c25680df02ff5f922d838bc62-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 923ece5c25680df02ff5f922d838bc62 2024-12-03T06:22:05,291 DEBUG [RS:1;122cddc95ec8:45577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:22:05,291 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T06:22:05,291 DEBUG [RS:0;122cddc95ec8:43629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T06:22:05,291 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T06:22:05,291 INFO [RS:1;122cddc95ec8:45577 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T06:22:05,291 INFO [RS:0;122cddc95ec8:43629 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T06:22:05,291 INFO [RS:0;122cddc95ec8:43629 {}] hbase.ChoreService(370): Chore service for: regionserver/122cddc95ec8:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-03T06:22:05,291 INFO [RS:1;122cddc95ec8:45577 {}] hbase.ChoreService(370): Chore service for: regionserver/122cddc95ec8:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-03T06:22:05,292 INFO [StoreOpener-923ece5c25680df02ff5f922d838bc62-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 923ece5c25680df02ff5f922d838bc62 columnFamilyName family 2024-12-03T06:22:05,292 INFO [RS:1;122cddc95ec8:45577 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T06:22:05,292 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T06:22:05,292 DEBUG [StoreOpener-923ece5c25680df02ff5f922d838bc62-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T06:22:05,292 INFO [regionserver/122cddc95ec8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T06:22:05,292 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T06:22:05,292 INFO [regionserver/122cddc95ec8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T06:22:05,292 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T06:22:05,292 INFO [RS:0;122cddc95ec8:43629 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T06:22:05,292 INFO [RS:0;122cddc95ec8:43629 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43629 2024-12-03T06:22:05,293 INFO [StoreOpener-923ece5c25680df02ff5f922d838bc62-1 {}] regionserver.HStore(327): Store=923ece5c25680df02ff5f922d838bc62/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T06:22:05,294 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for 923ece5c25680df02ff5f922d838bc62 2024-12-03T06:22:05,294 INFO [RS:1;122cddc95ec8:45577 {}] executor.ExecutorService(108): Executor-9-RS_OPEN_REGION-regionserver/122cddc95ec8:0 had [Event #37 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler), Event #38 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler)] on shutdown 2024-12-03T06:22:05,294 INFO [RS:1;122cddc95ec8:45577 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45577 2024-12-03T06:22:05,296 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for 923ece5c25680df02ff5f922d838bc62 2024-12-03T06:22:05,296 WARN [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(979): Failed initialize of region= test,vvv,1733206919969.923ece5c25680df02ff5f922d838bc62., starting to roll back memstore java.io.InterruptedIOException: Call interrupted at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy41.getFileInfo(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$getFileInfo$41(ClientNamenodeProtocolTranslatorPB.java:820) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getFileInfo(ClientNamenodeProtocolTranslatorPB.java:820) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy42.getFileInfo(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy45.getFileInfo(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy45.getFileInfo(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getFileInfo(DFSClient.java:1770) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1828) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1825) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1840) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem.exists(FileSystem.java:1862) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.wal.WALSplitUtil.getSplitEditFilesSorted(WALSplitUtil.java:298) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.replayRecoveredEditsIfAny(HRegion.java:5473) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.initializeRegionInternals(HRegion.java:1042) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:976) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7799) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegionFromTableDir(HRegion.java:7754) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7729) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7688) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7644) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:143) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:22:05,297 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(6241): Drop memstore for Store family in region test,vvv,1733206919969.923ece5c25680df02ff5f922d838bc62. , dropped memstoresize: [dataSize=0, getHeapSize=256, getOffHeapSize=0, getCellsCount=0 } 2024-12-03T06:22:05,298 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for 923ece5c25680df02ff5f922d838bc62: Running coprocessor pre-open hook at 1733206925287Writing region info on filesystem at 1733206925287Initializing all the Stores at 1733206925288 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733206925288Exception during region test,vvv,1733206919969.923ece5c25680df02ff5f922d838bc62. initialization. at 1733206925297 (+9 ms) 2024-12-03T06:22:05,298 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1722): Closing 923ece5c25680df02ff5f922d838bc62, disabling compactions & flushes 2024-12-03T06:22:05,298 INFO [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1755): Closing region test,vvv,1733206919969.923ece5c25680df02ff5f922d838bc62. 2024-12-03T06:22:05,298 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1733206919969.923ece5c25680df02ff5f922d838bc62. 2024-12-03T06:22:05,298 WARN [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1802): Interrupted while waiting for close lock on test,vvv,1733206919969.923ece5c25680df02ff5f922d838bc62. java.lang.InterruptedException: null at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireNanos(AbstractQueuedSynchronizer.java:993) ~[?:?] at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.tryLock(ReentrantReadWriteLock.java:1131) ~[?:?] at org.apache.hadoop.hbase.regionserver.HRegion.doClose(HRegion.java:1794) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1672) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1627) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1610) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7817) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegionFromTableDir(HRegion.java:7754) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7729) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7688) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7644) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:143) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:22:05,298 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1676): Region close journal for 923ece5c25680df02ff5f922d838bc62: Waiting for close lock at 1733206925298Running coprocessor pre-close hooks at 1733206925298Disabling compacts and flushes for region at 1733206925298 2024-12-03T06:22:05,298 WARN [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7819): Open region: {ENCODED => 923ece5c25680df02ff5f922d838bc62, NAME => 'test,vvv,1733206919969.923ece5c25680df02ff5f922d838bc62.', STARTKEY => 'vvv', ENDKEY => 'www'} failed. Try close region but got exception java.io.InterruptedIOException: Interrupted while waiting for close lock on test,vvv,1733206919969.923ece5c25680df02ff5f922d838bc62. at org.apache.hadoop.hbase.regionserver.HRegion.doClose(HRegion.java:1803) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1672) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1627) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1610) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7817) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegionFromTableDir(HRegion.java:7754) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7729) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7688) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7644) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:143) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireNanos(AbstractQueuedSynchronizer.java:993) ~[?:?] at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.tryLock(ReentrantReadWriteLock.java:1131) ~[?:?] at org.apache.hadoop.hbase.regionserver.HRegion.doClose(HRegion.java:1794) ~[classes/:?] ... 13 more 2024-12-03T06:22:05,298 WARN [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(84): Failed to open region test,vvv,1733206919969.923ece5c25680df02ff5f922d838bc62., will report to master java.io.InterruptedIOException: Call interrupted at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy41.getFileInfo(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$getFileInfo$41(ClientNamenodeProtocolTranslatorPB.java:820) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getFileInfo(ClientNamenodeProtocolTranslatorPB.java:820) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy42.getFileInfo(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy45.getFileInfo(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy45.getFileInfo(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getFileInfo(DFSClient.java:1770) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1828) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1825) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1840) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem.exists(FileSystem.java:1862) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.wal.WALSplitUtil.getSplitEditFilesSorted(WALSplitUtil.java:298) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.replayRecoveredEditsIfAny(HRegion.java:5473) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.initializeRegionInternals(HRegion.java:1042) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:976) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7799) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegionFromTableDir(HRegion.java:7754) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7729) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7688) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7644) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:143) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:22:05,299 WARN [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,vvv,1733206919969.923ece5c25680df02ff5f922d838bc62., aborting... java.io.IOException: Failed to report failed open to master: test,vvv,1733206919969.923ece5c25680df02ff5f922d838bc62. at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.cleanUpAndReportFailure(AssignRegionHandler.java:93) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:145) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T06:22:05,299 DEBUG [RS_OPEN_REGION-regionserver/122cddc95ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: Failed to open region test,vvv,1733206919969.923ece5c25680df02ff5f922d838bc62. and can not recover 2024-12-03T06:22:05,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T06:22:05,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/122cddc95ec8,43629,1733206917367 2024-12-03T06:22:05,302 INFO [RS:0;122cddc95ec8:43629 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T06:22:05,310 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/122cddc95ec8,45577,1733206917466 2024-12-03T06:22:05,310 INFO [RS:1;122cddc95ec8:45577 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T06:22:05,318 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [122cddc95ec8,43629,1733206917367] 2024-12-03T06:22:05,335 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/122cddc95ec8,43629,1733206917367 already deleted, retry=false 2024-12-03T06:22:05,335 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 122cddc95ec8,43629,1733206917367 expired; onlineServers=1 2024-12-03T06:22:05,335 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [122cddc95ec8,45577,1733206917466] 2024-12-03T06:22:05,343 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/122cddc95ec8,45577,1733206917466 already deleted, retry=false 2024-12-03T06:22:05,344 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 122cddc95ec8,45577,1733206917466 expired; onlineServers=0 2024-12-03T06:22:05,344 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '122cddc95ec8,35025,1733206916647' ***** 2024-12-03T06:22:05,344 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-03T06:22:05,344 INFO [M:0;122cddc95ec8:35025 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T06:22:05,344 INFO [M:0;122cddc95ec8:35025 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T06:22:05,344 DEBUG [M:0;122cddc95ec8:35025 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-03T06:22:05,344 DEBUG [M:0;122cddc95ec8:35025 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-03T06:22:05,344 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-03T06:22:05,345 DEBUG [master/122cddc95ec8:0:becomeActiveMaster-HFileCleaner.large.0-1733206918669 {}] cleaner.HFileCleaner(306): Exit Thread[master/122cddc95ec8:0:becomeActiveMaster-HFileCleaner.large.0-1733206918669,5,FailOnTimeoutGroup] 2024-12-03T06:22:05,344 DEBUG [master/122cddc95ec8:0:becomeActiveMaster-HFileCleaner.small.0-1733206918674 {}] cleaner.HFileCleaner(306): Exit Thread[master/122cddc95ec8:0:becomeActiveMaster-HFileCleaner.small.0-1733206918674,5,FailOnTimeoutGroup] 2024-12-03T06:22:05,345 INFO [M:0;122cddc95ec8:35025 {}] hbase.ChoreService(370): Chore service for: master/122cddc95ec8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-03T06:22:05,345 INFO [M:0;122cddc95ec8:35025 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T06:22:05,345 DEBUG [M:0;122cddc95ec8:35025 {}] master.HMaster(1795): Stopping service threads 2024-12-03T06:22:05,345 INFO [M:0;122cddc95ec8:35025 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-03T06:22:05,345 INFO [M:0;122cddc95ec8:35025 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T06:22:05,345 INFO [M:0;122cddc95ec8:35025 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T06:22:05,346 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-03T06:22:05,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-03T06:22:05,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T06:22:05,352 DEBUG [M:0;122cddc95ec8:35025 {}] zookeeper.ZKUtil(347): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-03T06:22:05,352 WARN [M:0;122cddc95ec8:35025 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-03T06:22:05,353 INFO [M:0;122cddc95ec8:35025 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/.lastflushedseqids 2024-12-03T06:22:05,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741870_1046 (size=1424) 2024-12-03T06:22:05,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741870_1046 (size=1424) 2024-12-03T06:22:05,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741870_1046 (size=1424) 2024-12-03T06:22:05,370 INFO [M:0;122cddc95ec8:35025 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T06:22:05,371 INFO [M:0;122cddc95ec8:35025 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-03T06:22:05,371 DEBUG [M:0;122cddc95ec8:35025 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T06:22:05,371 INFO [M:0;122cddc95ec8:35025 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T06:22:05,371 DEBUG [M:0;122cddc95ec8:35025 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T06:22:05,371 DEBUG [M:0;122cddc95ec8:35025 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T06:22:05,371 DEBUG [M:0;122cddc95ec8:35025 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T06:22:05,371 INFO [M:0;122cddc95ec8:35025 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=248.07 KB heapSize=301.46 KB 2024-12-03T06:22:05,386 DEBUG [M:0;122cddc95ec8:35025 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/22aa8c21f42b457fb643138dce091feb is 82, key is hbase:meta,,1/info:regioninfo/1733206923435/Put/seqid=0 2024-12-03T06:22:05,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741871_1047 (size=6071) 2024-12-03T06:22:05,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741871_1047 (size=6071) 2024-12-03T06:22:05,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741871_1047 (size=6071) 2024-12-03T06:22:05,394 INFO [M:0;122cddc95ec8:35025 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/22aa8c21f42b457fb643138dce091feb 2024-12-03T06:22:05,416 DEBUG [M:0;122cddc95ec8:35025 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c65f6da4bfe34dd5903d01df85fbc3a6 is 2987, key is \x00\x00\x00\x00\x00\x00\x009/proc:d/1733206925039/Put/seqid=0 2024-12-03T06:22:05,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T06:22:05,419 INFO [RS:0;122cddc95ec8:43629 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T06:22:05,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1009eb0b9ca0001, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T06:22:05,419 INFO [RS:0;122cddc95ec8:43629 {}] regionserver.HRegionServer(1031): Exiting; stopping=122cddc95ec8,43629,1733206917367; zookeeper connection closed. 2024-12-03T06:22:05,419 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@c936316 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@c936316 2024-12-03T06:22:05,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741872_1048 (size=12798) 2024-12-03T06:22:05,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741872_1048 (size=12798) 2024-12-03T06:22:05,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741872_1048 (size=12798) 2024-12-03T06:22:05,423 INFO [M:0;122cddc95ec8:35025 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=246.79 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c65f6da4bfe34dd5903d01df85fbc3a6 2024-12-03T06:22:05,427 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T06:22:05,427 INFO [RS:1;122cddc95ec8:45577 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T06:22:05,427 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45577-0x1009eb0b9ca0002, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T06:22:05,427 INFO [RS:1;122cddc95ec8:45577 {}] regionserver.HRegionServer(1031): Exiting; stopping=122cddc95ec8,45577,1733206917466; zookeeper connection closed. 2024-12-03T06:22:05,427 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1eab24a5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1eab24a5 2024-12-03T06:22:05,428 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-03T06:22:05,447 DEBUG [M:0;122cddc95ec8:35025 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/94323130c056494cbb31aa91941aad70 is 69, key is 122cddc95ec8,43629,1733206917367/rs:state/1733206918836/Put/seqid=0 2024-12-03T06:22:05,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741873_1049 (size=5442) 2024-12-03T06:22:05,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741873_1049 (size=5442) 2024-12-03T06:22:05,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741873_1049 (size=5442) 2024-12-03T06:22:05,456 INFO [M:0;122cddc95ec8:35025 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/94323130c056494cbb31aa91941aad70 2024-12-03T06:22:05,464 INFO [M:0;122cddc95ec8:35025 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 94323130c056494cbb31aa91941aad70 2024-12-03T06:22:05,480 DEBUG [M:0;122cddc95ec8:35025 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/303d6a60a50b4613b71a3652877a641b is 52, key is load_balancer_on/state:d/1733206922339/Put/seqid=0 2024-12-03T06:22:05,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741874_1050 (size=5058) 2024-12-03T06:22:05,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741874_1050 (size=5058) 2024-12-03T06:22:05,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741874_1050 (size=5058) 2024-12-03T06:22:05,487 INFO [M:0;122cddc95ec8:35025 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/303d6a60a50b4613b71a3652877a641b 2024-12-03T06:22:05,495 DEBUG [M:0;122cddc95ec8:35025 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/22aa8c21f42b457fb643138dce091feb as hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/22aa8c21f42b457fb643138dce091feb 2024-12-03T06:22:05,503 INFO [M:0;122cddc95ec8:35025 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/22aa8c21f42b457fb643138dce091feb, entries=14, sequenceid=569, filesize=5.9 K 2024-12-03T06:22:05,504 DEBUG [M:0;122cddc95ec8:35025 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c65f6da4bfe34dd5903d01df85fbc3a6 as hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c65f6da4bfe34dd5903d01df85fbc3a6 2024-12-03T06:22:05,511 INFO [M:0;122cddc95ec8:35025 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c65f6da4bfe34dd5903d01df85fbc3a6, entries=79, sequenceid=569, filesize=12.5 K 2024-12-03T06:22:05,512 DEBUG [M:0;122cddc95ec8:35025 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/94323130c056494cbb31aa91941aad70 as hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/94323130c056494cbb31aa91941aad70 2024-12-03T06:22:05,520 INFO [M:0;122cddc95ec8:35025 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 94323130c056494cbb31aa91941aad70 2024-12-03T06:22:05,520 INFO [M:0;122cddc95ec8:35025 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/94323130c056494cbb31aa91941aad70, entries=3, sequenceid=569, filesize=5.3 K 2024-12-03T06:22:05,521 DEBUG [M:0;122cddc95ec8:35025 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/303d6a60a50b4613b71a3652877a641b as hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/303d6a60a50b4613b71a3652877a641b 2024-12-03T06:22:05,528 INFO [M:0;122cddc95ec8:35025 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/3c029d87-b528-a491-4149-a1085c51f261/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/303d6a60a50b4613b71a3652877a641b, entries=1, sequenceid=569, filesize=4.9 K 2024-12-03T06:22:05,530 INFO [M:0;122cddc95ec8:35025 {}] regionserver.HRegion(3140): Finished flush of dataSize ~248.07 KB/254023, heapSize ~301.40 KB/308632, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 159ms, sequenceid=569, compaction requested=false 2024-12-03T06:22:05,531 INFO [M:0;122cddc95ec8:35025 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T06:22:05,532 DEBUG [M:0;122cddc95ec8:35025 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733206925371Disabling compacts and flushes for region at 1733206925371Disabling writes for close at 1733206925371Obtaining lock to block concurrent updates at 1733206925371Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733206925371Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=254023, getHeapSize=308632, getOffHeapSize=0, getCellsCount=708 at 1733206925371Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733206925372 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733206925372Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733206925386 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733206925386Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733206925400 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733206925416 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733206925416Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733206925430 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733206925446 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733206925446Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733206925464 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733206925480 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733206925480Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20e855c8: reopening flushed file at 1733206925493 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9cb937c: reopening flushed file at 1733206925503 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@309dcfa9: reopening flushed file at 1733206925511 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c771fed: reopening flushed file at 1733206925520 (+9 ms)Finished flush of dataSize ~248.07 KB/254023, heapSize ~301.40 KB/308632, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 159ms, sequenceid=569, compaction requested=false at 1733206925530 (+10 ms)Writing region close event to WAL at 1733206925531 (+1 ms)Closed at 1733206925531 2024-12-03T06:22:05,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32933 is added to blk_1073741830_1006 (size=292017) 2024-12-03T06:22:05,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40623 is added to blk_1073741830_1006 (size=292017) 2024-12-03T06:22:05,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741830_1006 (size=292017) 2024-12-03T06:22:05,535 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T06:22:05,535 INFO [M:0;122cddc95ec8:35025 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-03T06:22:05,535 INFO [M:0;122cddc95ec8:35025 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35025 2024-12-03T06:22:05,535 INFO [M:0;122cddc95ec8:35025 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T06:22:05,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T06:22:05,644 INFO [M:0;122cddc95ec8:35025 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T06:22:05,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35025-0x1009eb0b9ca0000, quorum=127.0.0.1:63630, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T06:22:05,682 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3c1f701{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T06:22:05,686 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T06:22:05,687 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T06:22:05,687 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T06:22:05,687 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/hadoop.log.dir/,STOPPED} 2024-12-03T06:22:05,691 WARN [BP-141962458-172.17.0.2-1733206912875 heartbeating to localhost/127.0.0.1:34349 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T06:22:05,691 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T06:22:05,691 WARN [BP-141962458-172.17.0.2-1733206912875 heartbeating to localhost/127.0.0.1:34349 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-141962458-172.17.0.2-1733206912875 (Datanode Uuid 90db776d-4a4a-401f-a7a8-677282c8abd6) service to localhost/127.0.0.1:34349 2024-12-03T06:22:05,691 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T06:22:05,692 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/cluster_9b3a36d6-da9f-80f6-d98c-a615b634f695/data/data5/current/BP-141962458-172.17.0.2-1733206912875 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T06:22:05,692 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/cluster_9b3a36d6-da9f-80f6-d98c-a615b634f695/data/data6/current/BP-141962458-172.17.0.2-1733206912875 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T06:22:05,693 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T06:22:05,694 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@34fcb585{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T06:22:05,695 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T06:22:05,695 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T06:22:05,695 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T06:22:05,695 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/hadoop.log.dir/,STOPPED} 2024-12-03T06:22:05,696 WARN [BP-141962458-172.17.0.2-1733206912875 heartbeating to localhost/127.0.0.1:34349 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T06:22:05,696 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T06:22:05,696 WARN [BP-141962458-172.17.0.2-1733206912875 heartbeating to localhost/127.0.0.1:34349 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-141962458-172.17.0.2-1733206912875 (Datanode Uuid 59069bac-f601-4e30-b0d8-e908363ae328) service to localhost/127.0.0.1:34349 2024-12-03T06:22:05,696 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T06:22:05,697 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/cluster_9b3a36d6-da9f-80f6-d98c-a615b634f695/data/data3/current/BP-141962458-172.17.0.2-1733206912875 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T06:22:05,697 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/cluster_9b3a36d6-da9f-80f6-d98c-a615b634f695/data/data4/current/BP-141962458-172.17.0.2-1733206912875 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T06:22:05,697 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T06:22:05,699 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@79b30fb0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T06:22:05,699 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T06:22:05,699 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T06:22:05,699 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T06:22:05,699 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/hadoop.log.dir/,STOPPED} 2024-12-03T06:22:05,700 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T06:22:05,700 WARN [BP-141962458-172.17.0.2-1733206912875 heartbeating to localhost/127.0.0.1:34349 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T06:22:05,700 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T06:22:05,700 WARN [BP-141962458-172.17.0.2-1733206912875 heartbeating to localhost/127.0.0.1:34349 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-141962458-172.17.0.2-1733206912875 (Datanode Uuid 1045b0d5-00e2-4e88-a577-6f1d9308a208) service to localhost/127.0.0.1:34349 2024-12-03T06:22:05,701 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/cluster_9b3a36d6-da9f-80f6-d98c-a615b634f695/data/data1/current/BP-141962458-172.17.0.2-1733206912875 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T06:22:05,701 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/cluster_9b3a36d6-da9f-80f6-d98c-a615b634f695/data/data2/current/BP-141962458-172.17.0.2-1733206912875 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T06:22:05,702 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T06:22:05,714 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64418798{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T06:22:05,715 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T06:22:05,715 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T06:22:05,716 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T06:22:05,716 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8980c1d0-489b-9f97-c27b-b83e18322d03/hadoop.log.dir/,STOPPED} 2024-12-03T06:22:05,727 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-03T06:22:05,774 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down